{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport tensorflow as tf\nimport tensorflow.keras.models as M\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.optimizers as O\nimport tensorflow.keras.losses as Loss\n\nfrom tqdm import tqdm\n\nfrom PIL import Image\nimport cv2\n\nimport matplotlib.pyplot as plt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n  # Restrict TensorFlow to only allocate 1GB of memory on the first GPU\n  try:\n    tf.config.experimental.set_virtual_device_configuration(\n        gpus[0],\n        [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=15240)])\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Virtual devices must be set before GPUs have been initialized\n    print(e)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 512\nEPOCHS = 10\nDIM =(100,100)\nMAX_LENGTH = 20","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '../input/bms-molecular-translation/train/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv(\"../input/bms-molecular-translation/train_labels.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(len(labels))):\n    labels.InChI.values[i] = labels.InChI.values[i][9:]\n    labels.InChI.values[i] = labels.InChI.values[i].split('/')[0]\n    image_id = labels.image_id.values[i]\n    labels.image_id.values[i] = train_path+image_id[0]+'/'+image_id[1]+'/'+image_id[2]+'/'+image_id+'.png'\nlabels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels['InChI'] = labels['InChI'].str.pad(width=MAX_LENGTH,side='right',fillchar='$')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"characters = set(char for label in labels.InChI.values for char in label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokens = dict()\n\nfor i in range(len(characters)):\n    tokens[list(characters)[i]] = i\n\ndetokens = {x:y for y,x in tokens.items()}\nvocab = len(tokens)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tokens)\nprint(detokens)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(len(labels))):    \n    labels.InChI.values[i] = ':'.join([str(tokens[x]) for x in labels.InChI.values[i]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread(labels.image_id.values[0],cv2.IMREAD_GRAYSCALE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(img)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(dim):\n    initializer = tf.keras.initializers.GlorotUniform()\n    inp = L.Input(shape=(dim[0],dim[1],1,),name = 'Input')\n    labels = L.Input(shape=(None,),name= 'Labels')\n    X = L.Conv2D(16,3,strides=1,name='Conv2D_1')(inp)\n    X = L.BatchNormalization(name='norm1')(X)\n    X = L.Activation('relu',name='relu_1')(X)\n    X = L.Conv2D(32,5,strides=1,name='Conv2D_2')(X)\n    X = L.BatchNormalization(name='norm2')(X)\n    X = L.Activation('relu',name='relu_2')(X)\n    X = L.Conv2D(64,7,strides=1,name='Conv2D_3')(X)\n    X = L.BatchNormalization(name='norm3')(X)\n    X = L.Activation('relu',name='relu_3')(X)\n    X = L.Conv2D(64,9,strides=1,name='Conv2D_4')(X)\n    X = L.BatchNormalization(name='norm4')(X)\n    X = L.Activation('relu',name='relu_4')(X)\n    X = L.MaxPooling2D(name='Max2D_1')(X)\n    X = L.Dropout(0.2,name='Dropout_1')(X)\n    inp2 = tf.image.resize(inp,[X.shape[1],X.shape[2]])\n    X = L.Add(name='Add_1')([X,inp2])\n    X = L.BatchNormalization(name='norm_A1')(X)\n    X = L.MaxPooling2D(name='Max2D_A1')(X)\n    X = L.Dropout(0.2,name='Dropout_A1')(X)\n    X = tf.reduce_sum(X,axis=3)\n    X = L.Bidirectional(L.LSTM(32,return_sequences=True,dropout=0.2))(X)\n    X = L.Bidirectional(L.LSTM(32,return_sequences=True,dropout=0.2))(X)\n    Out = L.Dense(vocab,activation='softmax',name='Output',kernel_initializer = initializer)(X)\n    model = M.Model(inputs=inp,outputs=Out)\n    adam = O.Adam(learning_rate=0.001)\n    model.compile(optimizer=adam,loss='categorical_crossentropy')\n    \n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model(DIM)\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model = M.load_model('../input/trained-model-for-bmsmolecular/phase1_base_model.h5')\nmodel.set_weights(base_model.get_weights())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model,to_file='model.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(label):\n    l = [tokens[x] for x in label]\n    return l\n\ndef data_generator(image_id,labels):\n    for j in range(len(labels)):\n        label = tf.cast(labels[j],dtype=tf.string)\n        label = tf.strings.split(label,sep=':')\n        label = tf.strings.to_number(label,out_type=tf.int32)\n        yield image_id[j],label\n\ndef preprocess_image(image_id,label):\n    image = tf.io.read_file(image_id)                            \n    image = tf.image.decode_png(image,channels=1)\n    image = tf.image.convert_image_dtype(image, tf.float32)\n    image = tf.image.resize(image,[DIM[0],DIM[1]])\n    label = preprocess_label(label)\n    return tf.data.Dataset.from_tensors((image, label))\n\ndef preprocess_label(label):\n    label = tf.one_hot(label,vocab,axis=-1)\n    return label\n\ndef preprocess_test_image(image_id):\n    image = tf.io.read_file(image_id)                            \n    image = tf.image.decode_png(image,channels=1)\n    image = tf.image.convert_image_dtype(image, tf.float32)\n    image = tf.image.resize_with_pad(image, DIM[0],DIM[1])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = tf.data.Dataset.from_generator(data_generator,args=[labels.image_id.values,labels.InChI.values],output_signature=(\n                    tf.TensorSpec(shape=(),dtype=tf.string),\n                    tf.TensorSpec(shape=(MAX_LENGTH,),dtype=tf.int32)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = dataset.interleave(lambda x,y: preprocess_image(x,y)).cache().batch(BATCH_SIZE,drop_remainder=True).repeat().prefetch(tf.data.AUTOTUNE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scheduler(epoch, lr):\n    if epoch < 8:\n        return lr\n    else:\n        return lr * tf.math.exp(-0.1)\n\nlrscheduler = tf.keras.callbacks.LearningRateScheduler(scheduler)\n\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(\"/content/drive/MyDrive/bms-molecular-translation/phase1-base-model-v1.4-{epoch:02d}-{val_loss:.4f}.hdf5\", monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n\nhistory = model.fit(train_data,\n                    epochs=EPOCHS,\n                    batch_size = BATCH_SIZE,\n                    steps_per_epoch=len(labels)//BATCH_SIZE,\n                    callbacks = [lrscheduler,checkpoint])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('phase1_base_model_v1.4.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}