{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":51294,"databundleVersionId":6923401,"sourceType":"competition"},{"sourceId":153412489,"sourceType":"kernelVersion"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/code/jordanyoung993/sequence-images-cnn/notebook","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\nimport pandas as pd\npd.set_option('display.max_columns', 30)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T11:59:36.207275Z","iopub.execute_input":"2023-12-04T11:59:36.208630Z","iopub.status.idle":"2023-12-04T11:59:36.669650Z","shell.execute_reply.started":"2023-12-04T11:59:36.208578Z","shell.execute_reply":"2023-12-04T11:59:36.668124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exp_2A3_MaP = pd.read_csv('/kaggle/input/srrnaf-4/2A3_MaP.csv')\nexp_2A3_MaP","metadata":{"execution":{"iopub.status.busy":"2023-12-04T11:59:36.671744Z","iopub.execute_input":"2023-12-04T11:59:36.672226Z","iopub.status.idle":"2023-12-04T12:00:43.667165Z","shell.execute_reply.started":"2023-12-04T11:59:36.672164Z","shell.execute_reply":"2023-12-04T12:00:43.665512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nTEMP_DIR = '/kaggle/temp'\n\nif not os.path.isdir(TEMP_DIR):\n    os.mkdir(TEMP_DIR)\nos.listdir('/kaggle')","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:00:43.669612Z","iopub.execute_input":"2023-12-04T12:00:43.670071Z","iopub.status.idle":"2023-12-04T12:00:43.680904Z","shell.execute_reply.started":"2023-12-04T12:00:43.670029Z","shell.execute_reply":"2023-12-04T12:00:43.679430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_types = ['2A3_MaP', 'DMS_MaP', 'test']\n\nfor experiment_type in experiment_types:\n    \n    exptype_dir = os.path.join(TEMP_DIR, experiment_type)\n    \n    if not os.path.isdir(exptype_dir):\n        os.mkdir(exptype_dir)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:00:43.684646Z","iopub.execute_input":"2023-12-04T12:00:43.685239Z","iopub.status.idle":"2023-12-04T12:00:43.696060Z","shell.execute_reply.started":"2023-12-04T12:00:43.685192Z","shell.execute_reply":"2023-12-04T12:00:43.694279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(TEMP_DIR)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:00:43.698039Z","iopub.execute_input":"2023-12-04T12:00:43.698589Z","iopub.status.idle":"2023-12-04T12:00:43.711033Z","shell.execute_reply.started":"2023-12-04T12:00:43.698538Z","shell.execute_reply":"2023-12-04T12:00:43.709406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_2A3_MaP = exp_2A3_MaP.drop(columns=exp_2A3_MaP.filter(like='error')) \\\n    .drop(columns=['dataset_name', 'reads', 'signal_to_noise', 'SN_filter'])","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:00:43.713289Z","iopub.execute_input":"2023-12-04T12:00:43.713730Z","iopub.status.idle":"2023-12-04T12:00:45.363686Z","shell.execute_reply.started":"2023-12-04T12:00:43.713696Z","shell.execute_reply":"2023-12-04T12:00:45.362512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reactivity_features = final_2A3_MaP.filter(like='reactivity').columns","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:00:45.365316Z","iopub.execute_input":"2023-12-04T12:00:45.365689Z","iopub.status.idle":"2023-12-04T12:00:45.786580Z","shell.execute_reply.started":"2023-12-04T12:00:45.365659Z","shell.execute_reply":"2023-12-04T12:00:45.785364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_2A3_MaP['reactivity'] = final_2A3_MaP[reactivity_features] \\\n    .apply(lambda i: i.fillna(i.mean()), axis=1) \\\n    .fillna(0.0) \\\n    .clip(lower=0.0, upper=1.0) \\\n    .values.tolist()\nfinal_2A3_MaP","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:00:45.788273Z","iopub.execute_input":"2023-12-04T12:00:45.788633Z","iopub.status.idle":"2023-12-04T12:05:09.845100Z","shell.execute_reply.started":"2023-12-04T12:00:45.788603Z","shell.execute_reply":"2023-12-04T12:05:09.843980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_png_path(i, experiment_type):\n    \n    return os.path.join(TEMP_DIR, experiment_type, f'{i}.png')","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:05:09.846960Z","iopub.execute_input":"2023-12-04T12:05:09.847581Z","iopub.status.idle":"2023-12-04T12:05:09.853950Z","shell.execute_reply.started":"2023-12-04T12:05:09.847541Z","shell.execute_reply":"2023-12-04T12:05:09.851990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_type = '2A3_MaP'\nfinal_2A3_MaP['png_path'] = final_2A3_MaP['sequence_id'] \\\n    .apply(generate_png_path, args=(experiment_type,))","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:05:09.857963Z","iopub.execute_input":"2023-12-04T12:05:09.858505Z","iopub.status.idle":"2023-12-04T12:05:12.964985Z","shell.execute_reply.started":"2023-12-04T12:05:09.858466Z","shell.execute_reply":"2023-12-04T12:05:12.963447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_2A3_MaP","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:05:12.966538Z","iopub.execute_input":"2023-12-04T12:05:12.966872Z","iopub.status.idle":"2023-12-04T12:05:13.494412Z","shell.execute_reply.started":"2023-12-04T12:05:12.966844Z","shell.execute_reply":"2023-12-04T12:05:13.493268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def pad_sequence(sequence, max_length=457, padding_value=\"Z\"):\n#     # Calculate how many padding values are needed\n#     padding_length = max_length - len(sequence)\n    \n#     # Create the padded sequence\n#     padded_sequence = list(sequence) + [padding_value] * padding_length\n    \n#     return padded_sequence\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequences = final_2A3_MaP[['sequence', 'png_path']].values.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:05:30.358490Z","iopub.execute_input":"2023-12-04T12:05:30.358887Z","iopub.status.idle":"2023-12-04T12:05:34.389888Z","shell.execute_reply.started":"2023-12-04T12:05:30.358858Z","shell.execute_reply":"2023-12-04T12:05:34.388557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I = 0\n\n# for sequence, path in sequences:\n    \n#     if I < 5:\n#         print(sequence)\n#         print(path)\n#         sequence = list(sequence) + ['Z'] * (457 - len(sequence))\n#         print(sequence)\n#         print(30 * '-')\n#         I += 1","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:18:06.287263Z","iopub.execute_input":"2023-12-04T12:18:06.287775Z","iopub.status.idle":"2023-12-04T12:18:06.293868Z","shell.execute_reply.started":"2023-12-04T12:18:06.287738Z","shell.execute_reply":"2023-12-04T12:18:06.292619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nitrogenous_bases = {'A': [1, 0, 0, 0, 0],\n                     'C': [0, 1, 0, 0, 0],\n                     'G': [0, 0, 1, 0, 0],\n                     'U': [0, 0, 0, 1, 0], \n                     'Z': [0, 0, 0, 0, 1]}","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:26:25.142400Z","iopub.execute_input":"2023-12-04T12:26:25.142929Z","iopub.status.idle":"2023-12-04T12:26:25.151220Z","shell.execute_reply.started":"2023-12-04T12:26:25.142890Z","shell.execute_reply":"2023-12-04T12:26:25.149467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom PIL import Image\nimport gc\nimport matplotlib.pyplot as plt\n\nfor sequence, path in sequences:\n    \n    sequence = list(sequence) + ['Z'] * (457 - len(sequence))\n    \n    letter_to_int8 = np.array([nitrogenous_bases[letter] for letter in sequence]).astype(np.uint8)\n    \n    img = Image.fromarray(letter_to_int8)\n    img.save(path)\n    \n    del sequence","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:29:46.636901Z","iopub.execute_input":"2023-12-04T12:29:46.637812Z","iopub.status.idle":"2023-12-04T12:41:43.278679Z","shell.execute_reply.started":"2023-12-04T12:29:46.637757Z","shell.execute_reply":"2023-12-04T12:41:43.277097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(letter_to_int8)\nplt.xlim((0.5))\nplt.ylim((0, 10))\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:47:15.595202Z","iopub.execute_input":"2023-12-04T12:47:15.595728Z","iopub.status.idle":"2023-12-04T12:47:15.907428Z","shell.execute_reply.started":"2023-12-04T12:47:15.595690Z","shell.execute_reply":"2023-12-04T12:47:15.905644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del letter_to_int8\ndel sequences\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:52:13.818161Z","iopub.execute_input":"2023-12-04T12:52:13.818660Z","iopub.status.idle":"2023-12-04T12:52:18.613243Z","shell.execute_reply.started":"2023-12-04T12:52:13.818626Z","shell.execute_reply":"2023-12-04T12:52:18.612138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('/kaggle/temp')","metadata":{"execution":{"iopub.status.busy":"2023-12-04T12:52:54.436959Z","iopub.execute_input":"2023-12-04T12:52:54.437464Z","iopub.status.idle":"2023-12-04T12:52:54.448157Z","shell.execute_reply.started":"2023-12-04T12:52:54.437427Z","shell.execute_reply":"2023-12-04T12:52:54.446781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_reactivity(sequence, max_length=457, padding_value=0.0):\n    \n    padding_length = max_length - len(sequence)\n    padded_sequence = sequence + [padding_value] * padding_length\n    \n    return padded_sequence","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:01:28.465072Z","iopub.execute_input":"2023-12-04T13:01:28.465610Z","iopub.status.idle":"2023-12-04T13:01:28.473729Z","shell.execute_reply.started":"2023-12-04T13:01:28.465574Z","shell.execute_reply":"2023-12-04T13:01:28.472379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_training_data(experiment_data):\n    experiment_data_train = experiment_data[['reactivity', 'png_path']]\n    \n    reactivity_feats = [f\"reactivity_{i}\" for i in range(457)]\n    experiment_data_train['reactivity'] = experiment_data_train['reactivity'].apply(pad_reactivity)\n    experiment_data_train[reactivity_feats] = pd.DataFrame(experiment_data_train.reactivity.tolist(), index= experiment_data_train.index)\n    experiment_data_train = experiment_data_train.drop(['reactivity'],axis=1)\n\n    del experiment_data\n    gc.collect()\n    \n    return experiment_data_train, reactivity_feats","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:01:31.598004Z","iopub.execute_input":"2023-12-04T13:01:31.598534Z","iopub.status.idle":"2023-12-04T13:01:31.607987Z","shell.execute_reply.started":"2023-12-04T13:01:31.598495Z","shell.execute_reply":"2023-12-04T13:01:31.606526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_data_train, reactivity_feats = prepare_training_data(final_2A3_MaP)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:01:33.893399Z","iopub.execute_input":"2023-12-04T13:01:33.893858Z","iopub.status.idle":"2023-12-04T13:04:52.997233Z","shell.execute_reply.started":"2023-12-04T13:01:33.893822Z","shell.execute_reply":"2023-12-04T13:04:52.996044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_data_train","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:05:07.969802Z","iopub.execute_input":"2023-12-04T13:05:07.970415Z","iopub.status.idle":"2023-12-04T13:05:08.408995Z","shell.execute_reply.started":"2023-12-04T13:05:07.970355Z","shell.execute_reply":"2023-12-04T13:05:08.407813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reactivity_feats[-10:]","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:05:55.970292Z","iopub.execute_input":"2023-12-04T13:05:55.970737Z","iopub.status.idle":"2023-12-04T13:05:55.981273Z","shell.execute_reply.started":"2023-12-04T13:05:55.970705Z","shell.execute_reply":"2023-12-04T13:05:55.979662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_test_val_split(experiment_data_train, train_frac, test_frac):\n    \n    train = experiment_data_train.sample(frac=train_frac, random_state=42)\n    test_val = experiment_data_train.drop(train.index)\n    test = test_val.sample(frac=test_frac, random_state=42)\n    valid = test_val.drop(test.index)\n    \n    del experiment_data_train\n    gc.collect()\n    \n    return train, test, valid","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:06:06.689328Z","iopub.execute_input":"2023-12-04T13:06:06.689751Z","iopub.status.idle":"2023-12-04T13:06:06.698904Z","shell.execute_reply.started":"2023-12-04T13:06:06.689719Z","shell.execute_reply":"2023-12-04T13:06:06.697117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, test, valid = train_test_val_split(experiment_data_train, 0.7, 0.66)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:06:07.595653Z","iopub.execute_input":"2023-12-04T13:06:07.596089Z","iopub.status.idle":"2023-12-04T13:06:18.553772Z","shell.execute_reply.started":"2023-12-04T13:06:07.596056Z","shell.execute_reply":"2023-12-04T13:06:18.552797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nBATCH_SIZE = 32\n\ndef build_image_generators(train, test, valid):\n    \n    datagen=tf.keras.preprocessing.image.ImageDataGenerator()\n\n    train_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=train,\n            x_col='png_path',\n            y_col=reactivity_feats,\n            class_mode='raw',\n            batch_size=BATCH_SIZE,\n            target_size=(457, 5),\n            shuffle=True\n    )\n\n    valid_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=valid,\n            x_col='png_path',\n            y_col=reactivity_feats,\n            class_mode='raw',\n            batch_size=BATCH_SIZE,\n            target_size=(457, 5),\n            shuffle=True\n    )\n\n    test_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=test,\n            x_col='png_path',\n            y_col=reactivity_feats,\n            target_size=(457, 5),\n            batch_size=BATCH_SIZE,\n            seed=42,\n            shuffle=False,\n            class_mode=None\n            )\n\n    del train\n    del valid\n    gc.collect()\n    \n    return train_generator, test_generator, valid_generator\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:12:06.208328Z","iopub.execute_input":"2023-12-04T13:12:06.208819Z","iopub.status.idle":"2023-12-04T13:12:06.221126Z","shell.execute_reply.started":"2023-12-04T13:12:06.208787Z","shell.execute_reply":"2023-12-04T13:12:06.219274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator, test_generator, valid_generator = build_image_generators(train, test, valid)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:12:07.017830Z","iopub.execute_input":"2023-12-04T13:12:07.018404Z","iopub.status.idle":"2023-12-04T13:12:41.196162Z","shell.execute_reply.started":"2023-12-04T13:12:07.018361Z","shell.execute_reply":"2023-12-04T13:12:41.194477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEARNING_RATE = 1e-3\n\ndef get_model():\n    \n    model = tf.keras.Sequential([\n        tf.keras.layers.Conv2D(2,(2,2), activation='relu'),\n        tf.keras.layers.MaxPooling2D(pool_size=(2,2)),\n\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='relu'),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='relu'),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='sigmoid')\n        \n    ])\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n            loss=tf.keras.losses.MeanSquaredError(),\n            )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:19:27.266312Z","iopub.execute_input":"2023-12-04T13:19:27.267734Z","iopub.status.idle":"2023-12-04T13:19:27.277923Z","shell.execute_reply.started":"2023-12-04T13:19:27.267691Z","shell.execute_reply":"2023-12-04T13:19:27.276057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 1\n\ndef fit_model(train_generator, valid_generator):\n    try:\n        del model\n        gc.collect()\n    except:\n        print(\"No Model Found\")\n\n    with tf.device('CPU'):\n\n        model = get_model()\n        history = model.fit(train_generator,validation_data=valid_generator, epochs=EPOCHS)\n        \n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:20:19.828690Z","iopub.execute_input":"2023-12-04T13:20:19.829207Z","iopub.status.idle":"2023-12-04T13:20:19.838032Z","shell.execute_reply.started":"2023-12-04T13:20:19.829156Z","shell.execute_reply":"2023-12-04T13:20:19.835981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2a3, history = fit_model(train_generator, valid_generator)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:20:20.276425Z","iopub.execute_input":"2023-12-04T13:20:20.276951Z","iopub.status.idle":"2023-12-04T13:38:40.191492Z","shell.execute_reply.started":"2023-12-04T13:20:20.276915Z","shell.execute_reply":"2023-12-04T13:38:40.189536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2a3.save_weights('/kaggle/working/checkpoints/2a3_model')","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:38:40.194870Z","iopub.execute_input":"2023-12-04T13:38:40.196248Z","iopub.status.idle":"2023-12-04T13:38:40.345616Z","shell.execute_reply.started":"2023-12-04T13:38:40.196144Z","shell.execute_reply":"2023-12-04T13:38:40.343802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}