{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7837703,"sourceType":"datasetVersion","datasetId":4594361},{"sourceId":7838717,"sourceType":"datasetVersion","datasetId":4595077},{"sourceId":7847709,"sourceType":"datasetVersion","datasetId":4601674}],"dockerImageVersionId":30666,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nfrom tensorflow import keras\nfrom keras import layers\n\n# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n\n# instantiate a distribution strategy\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.TPUStrategy(tpu)\n\ndata1 = np.load('/kaggle/input/y-log-scale-1/Y_log_scale_1.npy')\ndata2 = np.load('/kaggle/input/y-log-scale-2/Y_log_scale_2.npy')\nvote_columns_normalized = np.load('/kaggle/input/vote-columns-normalized/vote_columns_normalized.npy')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-25T03:14:03.525224Z","iopub.execute_input":"2024-05-25T03:14:03.525568Z","iopub.status.idle":"2024-05-25T03:18:14.979821Z","shell.execute_reply.started":"2024-05-25T03:14:03.525529Z","shell.execute_reply":"2024-05-25T03:18:14.978577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrograms = np.concatenate((data1, data2))","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:19:35.73081Z","iopub.execute_input":"2024-05-25T03:19:35.731267Z","iopub.status.idle":"2024-05-25T03:19:47.246332Z","shell.execute_reply.started":"2024-05-25T03:19:35.731231Z","shell.execute_reply":"2024-05-25T03:19:47.245191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrograms.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:19:50.98638Z","iopub.execute_input":"2024-05-25T03:19:50.986816Z","iopub.status.idle":"2024-05-25T03:19:50.995408Z","shell.execute_reply.started":"2024-05-25T03:19:50.986782Z","shell.execute_reply":"2024-05-25T03:19:50.994351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrograms = np.moveaxis(spectrograms, -1, 1)","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:19:52.354767Z","iopub.execute_input":"2024-05-25T03:19:52.355167Z","iopub.status.idle":"2024-05-25T03:19:52.359533Z","shell.execute_reply.started":"2024-05-25T03:19:52.355136Z","shell.execute_reply":"2024-05-25T03:19:52.358523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrograms.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:19:55.018265Z","iopub.execute_input":"2024-05-25T03:19:55.018736Z","iopub.status.idle":"2024-05-25T03:19:55.024873Z","shell.execute_reply.started":"2024-05-25T03:19:55.018697Z","shell.execute_reply":"2024-05-25T03:19:55.023901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrograms = spectrograms.reshape(-1, 81, 19 * 51)","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:19:56.67286Z","iopub.execute_input":"2024-05-25T03:19:56.67331Z","iopub.status.idle":"2024-05-25T03:19:56.677911Z","shell.execute_reply.started":"2024-05-25T03:19:56.673267Z","shell.execute_reply":"2024-05-25T03:19:56.676913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrograms.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:19:57.69317Z","iopub.execute_input":"2024-05-25T03:19:57.694217Z","iopub.status.idle":"2024-05-25T03:19:57.699489Z","shell.execute_reply.started":"2024-05-25T03:19:57.694175Z","shell.execute_reply":"2024-05-25T03:19:57.698537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate random permutation\npermutation = np.random.permutation(len(spectrograms))","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:19:59.814272Z","iopub.execute_input":"2024-05-25T03:19:59.814688Z","iopub.status.idle":"2024-05-25T03:19:59.821322Z","shell.execute_reply.started":"2024-05-25T03:19:59.814654Z","shell.execute_reply":"2024-05-25T03:19:59.820377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shuffle both arrays using the permutation\nshuffled_spectrograms = spectrograms[permutation]\nshuffled_vote_columns_normalized = vote_columns_normalized[permutation]","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:20:00.808874Z","iopub.execute_input":"2024-05-25T03:20:00.809323Z","iopub.status.idle":"2024-05-25T03:20:10.421244Z","shell.execute_reply.started":"2024-05-25T03:20:00.809288Z","shell.execute_reply":"2024-05-25T03:20:10.420008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrograms_val = shuffled_spectrograms[0:int(shuffled_spectrograms.shape[0]/10)]\nspectrograms_train = shuffled_spectrograms[int(shuffled_spectrograms.shape[0]/10):]","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:21:04.463957Z","iopub.execute_input":"2024-05-25T03:21:04.464425Z","iopub.status.idle":"2024-05-25T03:21:04.469498Z","shell.execute_reply.started":"2024-05-25T03:21:04.46439Z","shell.execute_reply":"2024-05-25T03:21:04.468533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(spectrograms_val.shape)\nprint(spectrograms_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:21:05.405123Z","iopub.execute_input":"2024-05-25T03:21:05.406089Z","iopub.status.idle":"2024-05-25T03:21:05.411674Z","shell.execute_reply.started":"2024-05-25T03:21:05.406041Z","shell.execute_reply":"2024-05-25T03:21:05.410387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vote_columns_normalized_val = shuffled_vote_columns_normalized[0:int(shuffled_vote_columns_normalized.shape[0]/10)]\nvote_columns_normalized_train = shuffled_vote_columns_normalized[int(shuffled_vote_columns_normalized.shape[0]/10):]","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:21:06.319165Z","iopub.execute_input":"2024-05-25T03:21:06.319593Z","iopub.status.idle":"2024-05-25T03:21:06.324659Z","shell.execute_reply.started":"2024-05-25T03:21:06.319563Z","shell.execute_reply":"2024-05-25T03:21:06.323416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(vote_columns_normalized_val.shape)\nprint(vote_columns_normalized_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:21:07.323551Z","iopub.execute_input":"2024-05-25T03:21:07.323946Z","iopub.status.idle":"2024-05-25T03:21:07.329073Z","shell.execute_reply.started":"2024-05-25T03:21:07.323915Z","shell.execute_reply":"2024-05-25T03:21:07.327927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's make a list of CONSTANTS for modelling:\nLAYERS = [8, 8, 8, 6] # number of units in hidden layers and output layers\n\nM_TRAIN = spectrograms_train.shape[0] # number of training examples\n\nM_VAL = spectrograms_val.shape[0] # number of validation examples\n\nT = spectrograms_train.shape[1] # sequence length (timesteps)\n\nN = spectrograms_train.shape[2] # number of features\n\nBATCH = M_TRAIN # batch size\n\nLR = 1e-3 # learning rate\n\nEPOCH = 10 # number of epochs\n\nLAMBD = 3e-2 # lambda in L2 regularization\n\nDP = 0.0 # dropout rate\n\nRDP = 0.2 # recurrent dropout rate","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:41:18.35552Z","iopub.execute_input":"2024-05-25T03:41:18.355901Z","iopub.status.idle":"2024-05-25T03:41:18.362801Z","shell.execute_reply.started":"2024-05-25T03:41:18.35587Z","shell.execute_reply":"2024-05-25T03:41:18.361219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, LSTM, BatchNormalization\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom keras.regularizers import l2\nfrom time import time","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:41:19.332598Z","iopub.execute_input":"2024-05-25T03:41:19.333707Z","iopub.status.idle":"2024-05-25T03:41:19.338131Z","shell.execute_reply.started":"2024-05-25T03:41:19.333667Z","shell.execute_reply":"2024-05-25T03:41:19.33707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(LR))","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:41:22.114659Z","iopub.execute_input":"2024-05-25T03:41:22.115046Z","iopub.status.idle":"2024-05-25T03:41:22.119845Z","shell.execute_reply.started":"2024-05-25T03:41:22.115014Z","shell.execute_reply":"2024-05-25T03:41:22.118892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build the Model\nwith tpu_strategy.scope():\n    model = Sequential()\n    model.add(LSTM(input_shape=(T,N),\n                   units=LAYERS[0],\n                   activation='tanh',\n                   recurrent_activation='hard_sigmoid',\n                   kernel_regularizer=l2(LAMBD),\n                   recurrent_regularizer=l2(LAMBD),\n                   dropout=DP,\n                   recurrent_dropout=RDP,\n                   return_sequences=True,\n                   return_state=False,\n                   stateful=False,\n                   unroll=False\n             ))\n    model.add(BatchNormalization())\n    \n    model.add(LSTM(units=LAYERS[1],\n               activation='tanh',\n               recurrent_activation='hard_sigmoid',\n               kernel_regularizer=l2(LAMBD),\n               recurrent_regularizer=l2(LAMBD),\n               dropout=DP,\n               recurrent_dropout=RDP,\n               return_sequences=True,\n               return_state=False,\n               stateful=False,\n               unroll=False\n            ))\n    model.add(BatchNormalization())\n    \n    model.add(LSTM(units=LAYERS[2],\n               activation='tanh',\n               recurrent_activation='hard_sigmoid',\n               kernel_regularizer=l2(LAMBD),\n               recurrent_regularizer=l2(LAMBD),\n               dropout=DP,\n               recurrent_dropout=RDP,\n               return_sequences=False,\n               return_state=False,\n               stateful=False,\n               unroll=False\n            ))\n    model.add(BatchNormalization())\n    \n    model.add(Dense(units=6, activation='softmax'))\n    \n    # Compile the model with Adam optimizer\n    model.compile(loss='categorical_crossentropy',\n              metrics=[tf.keras.metrics.kl_divergence],\n              optimizer=Adam(learning_rate=LR ))","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:42:22.609822Z","iopub.execute_input":"2024-05-25T03:42:22.610221Z","iopub.status.idle":"2024-05-25T03:42:22.992564Z","shell.execute_reply.started":"2024-05-25T03:42:22.610187Z","shell.execute_reply":"2024-05-25T03:42:22.991346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a learning rate decay method:\nlr_decay = ReduceLROnPlateau(monitor='loss', \n                             patience=1, verbose=0, \n                             factor=0.5, min_lr=1e-8)","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:42:27.564522Z","iopub.execute_input":"2024-05-25T03:42:27.564891Z","iopub.status.idle":"2024-05-25T03:42:27.569587Z","shell.execute_reply.started":"2024-05-25T03:42:27.564862Z","shell.execute_reply":"2024-05-25T03:42:27.568488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Early Stopping:\nearly_stop = EarlyStopping(monitor='val_acc', min_delta=0, \n                           patience=30, verbose=1, mode='auto',\n                           baseline=0, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:42:28.496377Z","iopub.execute_input":"2024-05-25T03:42:28.496794Z","iopub.status.idle":"2024-05-25T03:42:28.501776Z","shell.execute_reply.started":"2024-05-25T03:42:28.496758Z","shell.execute_reply":"2024-05-25T03:42:28.500735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(spectrograms_train,\n          vote_columns_normalized_train,\n          steps_per_epoch = int(spectrograms_train.shape[0] / 128),\n          epochs = 50,\n          validation_data=(spectrograms_val, vote_columns_normalized_val),\n          verbose=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-25T03:46:04.22281Z","iopub.execute_input":"2024-05-25T03:46:04.223246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hola mundo","metadata":{"execution":{"iopub.status.busy":"2024-05-25T04:02:26.743466Z","iopub.execute_input":"2024-05-25T04:02:26.743845Z","iopub.status.idle":"2024-05-25T04:02:26.74783Z","shell.execute_reply.started":"2024-05-25T04:02:26.743801Z","shell.execute_reply":"2024-05-25T04:02:26.746946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#","metadata":{"execution":{"iopub.status.busy":"2024-05-25T04:02:26.749162Z","iopub.execute_input":"2024-05-25T04:02:26.749583Z","iopub.status.idle":"2024-05-25T04:02:26.760544Z","shell.execute_reply.started":"2024-05-25T04:02:26.749554Z","shell.execute_reply":"2024-05-25T04:02:26.75975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#","metadata":{"execution":{"iopub.status.busy":"2024-05-25T04:02:26.761986Z","iopub.execute_input":"2024-05-25T04:02:26.762259Z","iopub.status.idle":"2024-05-25T04:02:26.779056Z","shell.execute_reply.started":"2024-05-25T04:02:26.762231Z","shell.execute_reply":"2024-05-25T04:02:26.77824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#","metadata":{"execution":{"iopub.status.busy":"2024-05-25T04:02:26.780328Z","iopub.execute_input":"2024-05-25T04:02:26.780622Z","iopub.status.idle":"2024-05-25T04:02:26.79727Z","shell.execute_reply.started":"2024-05-25T04:02:26.780594Z","shell.execute_reply":"2024-05-25T04:02:26.796413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#","metadata":{"execution":{"iopub.status.busy":"2024-05-25T04:02:42.976852Z","iopub.execute_input":"2024-05-25T04:02:42.977573Z","iopub.status.idle":"2024-05-25T04:02:42.981258Z","shell.execute_reply.started":"2024-05-25T04:02:42.977534Z","shell.execute_reply":"2024-05-25T04:02:42.980324Z"},"trusted":true},"execution_count":null,"outputs":[]}]}