{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":51294,"databundleVersionId":6923401,"sourceType":"competition"},{"sourceId":153412489,"sourceType":"kernelVersion"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/code/jordanyoung993/sequence-images-cnn/notebook","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/jordanyoung993/cnn-submission/notebook","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\nimport pandas as pd\npd.set_option('display.max_columns', 30)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:29:39.753908Z","iopub.execute_input":"2023-12-04T14:29:39.754237Z","iopub.status.idle":"2023-12-04T14:29:40.042833Z","shell.execute_reply.started":"2023-12-04T14:29:39.754212Z","shell.execute_reply":"2023-12-04T14:29:40.041940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exp_DMS_MaP = pd.read_csv('/kaggle/input/srrnaf-4/DMS_MaP.csv')\nexp_DMS_MaP","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:29:40.044695Z","iopub.execute_input":"2023-12-04T14:29:40.045124Z","iopub.status.idle":"2023-12-04T14:30:18.517348Z","shell.execute_reply.started":"2023-12-04T14:29:40.045095Z","shell.execute_reply":"2023-12-04T14:30:18.516473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nTEMP_DIR = '/kaggle/temp'\n\nif not os.path.isdir(TEMP_DIR):\n    os.mkdir(TEMP_DIR)\nos.listdir('/kaggle')","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:30:18.518474Z","iopub.execute_input":"2023-12-04T14:30:18.518733Z","iopub.status.idle":"2023-12-04T14:30:18.525550Z","shell.execute_reply.started":"2023-12-04T14:30:18.518711Z","shell.execute_reply":"2023-12-04T14:30:18.524772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_types = ['2A3_MaP', 'DMS_MaP', 'test']\n\nfor experiment_type in experiment_types:\n    \n    exptype_dir = os.path.join(TEMP_DIR, experiment_type)\n    \n    if not os.path.isdir(exptype_dir):\n        os.mkdir(exptype_dir)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:30:18.527317Z","iopub.execute_input":"2023-12-04T14:30:18.527540Z","iopub.status.idle":"2023-12-04T14:30:18.542055Z","shell.execute_reply.started":"2023-12-04T14:30:18.527521Z","shell.execute_reply":"2023-12-04T14:30:18.541173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(TEMP_DIR)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:30:18.543350Z","iopub.execute_input":"2023-12-04T14:30:18.543672Z","iopub.status.idle":"2023-12-04T14:30:18.556704Z","shell.execute_reply.started":"2023-12-04T14:30:18.543649Z","shell.execute_reply":"2023-12-04T14:30:18.555977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_DMS_MaP = exp_DMS_MaP.drop(columns=exp_DMS_MaP.filter(like='error')) \\\n    .drop(columns=['dataset_name', 'reads', 'signal_to_noise', 'SN_filter'])","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:30:18.557653Z","iopub.execute_input":"2023-12-04T14:30:18.558023Z","iopub.status.idle":"2023-12-04T14:30:19.052637Z","shell.execute_reply.started":"2023-12-04T14:30:18.558002Z","shell.execute_reply":"2023-12-04T14:30:19.051775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reactivity_features = final_DMS_MaP.filter(like='reactivity').columns","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:30:19.053532Z","iopub.execute_input":"2023-12-04T14:30:19.053805Z","iopub.status.idle":"2023-12-04T14:30:19.218875Z","shell.execute_reply.started":"2023-12-04T14:30:19.053783Z","shell.execute_reply":"2023-12-04T14:30:19.217877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_DMS_MaP['reactivity'] = final_DMS_MaP[reactivity_features] \\\n    .apply(lambda i: i.fillna(i.mean()), axis=1) \\\n    .fillna(0.0) \\\n    .clip(lower=0.0, upper=1.0) \\\n    .values.tolist()\nfinal_DMS_MaP","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:30:19.220162Z","iopub.execute_input":"2023-12-04T14:30:19.221073Z","iopub.status.idle":"2023-12-04T14:33:21.001998Z","shell.execute_reply.started":"2023-12-04T14:30:19.221033Z","shell.execute_reply":"2023-12-04T14:33:21.001320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_png_path(i, experiment_type):\n    \n    return os.path.join(TEMP_DIR, experiment_type, f'{i}.png')","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:33:21.003109Z","iopub.execute_input":"2023-12-04T14:33:21.003350Z","iopub.status.idle":"2023-12-04T14:33:21.007794Z","shell.execute_reply.started":"2023-12-04T14:33:21.003330Z","shell.execute_reply":"2023-12-04T14:33:21.006843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_type = 'DMS_MaP'\nfinal_DMS_MaP['png_path'] = final_DMS_MaP['sequence_id'] \\\n    .apply(generate_png_path, args=(experiment_type,))","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:33:21.011077Z","iopub.execute_input":"2023-12-04T14:33:21.011336Z","iopub.status.idle":"2023-12-04T14:33:22.680030Z","shell.execute_reply.started":"2023-12-04T14:33:21.011308Z","shell.execute_reply":"2023-12-04T14:33:22.679008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_DMS_MaP","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:33:22.681147Z","iopub.execute_input":"2023-12-04T14:33:22.681365Z","iopub.status.idle":"2023-12-04T14:33:22.984973Z","shell.execute_reply.started":"2023-12-04T14:33:22.681345Z","shell.execute_reply":"2023-12-04T14:33:22.984148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequences = final_DMS_MaP[['sequence', 'png_path']].values.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:33:22.986166Z","iopub.execute_input":"2023-12-04T14:33:22.986499Z","iopub.status.idle":"2023-12-04T14:33:23.180282Z","shell.execute_reply.started":"2023-12-04T14:33:22.986470Z","shell.execute_reply":"2023-12-04T14:33:23.179412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nitrogenous_bases = {'A': [1, 0, 0, 0, 0],\n                     'C': [0, 1, 0, 0, 0],\n                     'G': [0, 0, 1, 0, 0],\n                     'U': [0, 0, 0, 1, 0], \n                     'Z': [0, 0, 0, 0, 1]}","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:33:23.181750Z","iopub.execute_input":"2023-12-04T14:33:23.182151Z","iopub.status.idle":"2023-12-04T14:33:23.187541Z","shell.execute_reply.started":"2023-12-04T14:33:23.182124Z","shell.execute_reply":"2023-12-04T14:33:23.186320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom PIL import Image\nimport gc\n\nfor sequence, path in sequences:\n    \n    sequence = list(sequence) + ['Z'] * (457 - len(sequence))\n    \n    letter_to_int8 = np.array([nitrogenous_bases[letter] for letter in sequence]).astype(np.uint8)\n    \n    img = Image.fromarray(letter_to_int8)\n    img.save(path)\n    \n    del sequence","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:33:23.189244Z","iopub.execute_input":"2023-12-04T14:33:23.189860Z","iopub.status.idle":"2023-12-04T14:39:35.630839Z","shell.execute_reply.started":"2023-12-04T14:33:23.189830Z","shell.execute_reply":"2023-12-04T14:39:35.629380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.imshow(letter_to_int8)\nplt.xlim((0.5))\nplt.ylim((0, 10))","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:39:35.632705Z","iopub.execute_input":"2023-12-04T14:39:35.633003Z","iopub.status.idle":"2023-12-04T14:39:35.814148Z","shell.execute_reply.started":"2023-12-04T14:39:35.632980Z","shell.execute_reply":"2023-12-04T14:39:35.813279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del letter_to_int8\ndel sequences\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:39:35.815133Z","iopub.execute_input":"2023-12-04T14:39:35.815370Z","iopub.status.idle":"2023-12-04T14:39:37.631037Z","shell.execute_reply.started":"2023-12-04T14:39:35.815349Z","shell.execute_reply":"2023-12-04T14:39:37.630119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_reactivity(sequence, max_length=457, padding_value=0.0):\n    \n    padding_length = max_length - len(sequence)\n    padded_sequence = sequence + [padding_value] * padding_length\n    \n    return padded_sequence","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:39:37.632271Z","iopub.execute_input":"2023-12-04T14:39:37.632517Z","iopub.status.idle":"2023-12-04T14:39:37.639269Z","shell.execute_reply.started":"2023-12-04T14:39:37.632495Z","shell.execute_reply":"2023-12-04T14:39:37.638504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_training_data(experiment_data):\n    experiment_data_train = experiment_data[['reactivity', 'png_path']]\n    \n    reactivity_feats = [f\"reactivity_{i}\" for i in range(457)]\n    experiment_data_train['reactivity'] = experiment_data_train['reactivity'].apply(pad_reactivity)\n    experiment_data_train[reactivity_feats] = pd.DataFrame(experiment_data_train.reactivity.tolist(), index= experiment_data_train.index)\n    experiment_data_train = experiment_data_train.drop(['reactivity'],axis=1)\n\n    del experiment_data\n    gc.collect()\n    \n    return experiment_data_train, reactivity_feats","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:39:37.640343Z","iopub.execute_input":"2023-12-04T14:39:37.640601Z","iopub.status.idle":"2023-12-04T14:39:37.650222Z","shell.execute_reply.started":"2023-12-04T14:39:37.640555Z","shell.execute_reply":"2023-12-04T14:39:37.649278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_data_train, reactivity_feats = prepare_training_data(final_DMS_MaP)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:39:37.652180Z","iopub.execute_input":"2023-12-04T14:39:37.652628Z","iopub.status.idle":"2023-12-04T14:40:27.224256Z","shell.execute_reply.started":"2023-12-04T14:39:37.652591Z","shell.execute_reply":"2023-12-04T14:40:27.223254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_data_train","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:27.225405Z","iopub.execute_input":"2023-12-04T14:40:27.225661Z","iopub.status.idle":"2023-12-04T14:40:27.440884Z","shell.execute_reply.started":"2023-12-04T14:40:27.225639Z","shell.execute_reply":"2023-12-04T14:40:27.440093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reactivity_feats[-10:]","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:27.442321Z","iopub.execute_input":"2023-12-04T14:40:27.442837Z","iopub.status.idle":"2023-12-04T14:40:27.448912Z","shell.execute_reply.started":"2023-12-04T14:40:27.442809Z","shell.execute_reply":"2023-12-04T14:40:27.447952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_test_val_split(experiment_data_train, train_frac, test_frac):\n    \n    train = experiment_data_train.sample(frac=train_frac, random_state=42)\n    test_val = experiment_data_train.drop(train.index)\n    test = test_val.sample(frac=test_frac, random_state=42)\n    valid = test_val.drop(test.index)\n    \n    del experiment_data_train\n    gc.collect()\n    \n    return train, test, valid","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:27.450042Z","iopub.execute_input":"2023-12-04T14:40:27.450515Z","iopub.status.idle":"2023-12-04T14:40:27.460168Z","shell.execute_reply.started":"2023-12-04T14:40:27.450492Z","shell.execute_reply":"2023-12-04T14:40:27.459529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, test, valid = train_test_val_split(experiment_data_train, 0.7, 0.66)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:27.460936Z","iopub.execute_input":"2023-12-04T14:40:27.461541Z","iopub.status.idle":"2023-12-04T14:40:32.896406Z","shell.execute_reply.started":"2023-12-04T14:40:27.461518Z","shell.execute_reply":"2023-12-04T14:40:32.895250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nBATCH_SIZE = 32\n\ndef build_image_generators(train, test, valid):\n    \n    datagen=tf.keras.preprocessing.image.ImageDataGenerator()\n\n    train_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=train,\n            x_col='png_path',\n            y_col=reactivity_feats,\n            class_mode='raw',\n            batch_size=BATCH_SIZE,\n            target_size=(457, 5),\n            shuffle=True\n    )\n\n    valid_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=valid,\n            x_col='png_path',\n            y_col=reactivity_feats,\n            class_mode='raw',\n            batch_size=BATCH_SIZE,\n            target_size=(457, 5),\n            shuffle=True\n    )\n\n    test_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=test,\n            x_col='png_path',\n            y_col=reactivity_feats,\n            target_size=(457, 5),\n            batch_size=BATCH_SIZE,\n            seed=42,\n            shuffle=False,\n            class_mode=None\n            )\n\n    del train\n    del valid\n    gc.collect()\n    \n    return train_generator, test_generator, valid_generator","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:32.897800Z","iopub.execute_input":"2023-12-04T14:40:32.898086Z","iopub.status.idle":"2023-12-04T14:40:43.881174Z","shell.execute_reply.started":"2023-12-04T14:40:32.898065Z","shell.execute_reply":"2023-12-04T14:40:43.880277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator, test_generator, valid_generator = build_image_generators(train, test, valid)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:43.882358Z","iopub.execute_input":"2023-12-04T14:40:43.882862Z","iopub.status.idle":"2023-12-04T14:40:58.838169Z","shell.execute_reply.started":"2023-12-04T14:40:43.882839Z","shell.execute_reply":"2023-12-04T14:40:58.837245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEARNING_RATE = 1e-3\n\ndef get_model():\n    \n    model = tf.keras.Sequential([\n        tf.keras.layers.Conv2D(2,(2,2), activation='relu'),\n        tf.keras.layers.MaxPooling2D(pool_size=(2,2)),\n\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='relu'),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='relu'),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='sigmoid')\n        \n    ])\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n            loss=tf.keras.losses.MeanSquaredError(),\n            )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:58.839509Z","iopub.execute_input":"2023-12-04T14:40:58.840205Z","iopub.status.idle":"2023-12-04T14:40:58.846538Z","shell.execute_reply.started":"2023-12-04T14:40:58.840175Z","shell.execute_reply":"2023-12-04T14:40:58.845976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 1\n\ndef fit_model(train_generator, valid_generator):\n    try:\n        del model\n        gc.collect()\n    except:\n        print(\"No Model Found\")\n\n    with tf.device('CPU'):\n\n        model = get_model()\n        history = model.fit(train_generator,validation_data=valid_generator, epochs=EPOCHS)\n        \n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:58.847423Z","iopub.execute_input":"2023-12-04T14:40:58.847901Z","iopub.status.idle":"2023-12-04T14:40:58.862526Z","shell.execute_reply.started":"2023-12-04T14:40:58.847875Z","shell.execute_reply":"2023-12-04T14:40:58.861952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_DMS, history = fit_model(train_generator, valid_generator)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:40:58.866106Z","iopub.execute_input":"2023-12-04T14:40:58.866529Z","iopub.status.idle":"2023-12-04T14:51:21.963642Z","shell.execute_reply.started":"2023-12-04T14:40:58.866500Z","shell.execute_reply":"2023-12-04T14:51:21.962635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_DMS.save_weights('/kaggle/working/checkpoints/DMS_model')","metadata":{"execution":{"iopub.status.busy":"2023-12-04T14:51:21.964879Z","iopub.execute_input":"2023-12-04T14:51:21.965154Z","iopub.status.idle":"2023-12-04T14:51:22.029156Z","shell.execute_reply.started":"2023-12-04T14:51:21.965133Z","shell.execute_reply":"2023-12-04T14:51:22.028229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}