{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport gc\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom sklearn.metrics import mean_squared_error\n\n# To ignore warinings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# constants \nTEMP_DIR = \"/kaggle/temp\"\nTESTING = True\nPLOT_RESULTS = True\nTRAIN = True\nEVAL = True\n\n\n# Model Hyperparameters\nEPOCHS = 1\nBATCH_SIZE = 32\nLEARNING_RATE = 1e-3\n","metadata":{"execution":{"iopub.status.busy":"2023-11-08T23:26:55.727719Z","iopub.execute_input":"2023-11-08T23:26:55.728059Z","iopub.status.idle":"2023-11-08T23:26:55.734274Z","shell.execute_reply.started":"2023-11-08T23:26:55.728035Z","shell.execute_reply":"2023-11-08T23:26:55.733310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef split_dataset():\n    dataset = pd.read_csv(\"/kaggle/input/stanford-ribonanza-rna-folding/train_data.csv\")\n    experiments = [\"DMS_MaP\",\"2A3_MaP\"]\n\n    for experiment in experiments:\n        experiment_data = dataset[dataset['experiment_type'] == experiment]\n        experiment_data.to_csv(f\"./{experiment}.csv\",index=False)\n    \n        del experiment_data\n    del dataset\n    del experiments\n    gc.collect()\n\ndef setup_directories():\n    \n    # setup temporary directory \n    if not os.path.isdir(TEMP_DIR):\n        os.mkdir(TEMP_DIR)\n    os.listdir(\"/kaggle\")\n\n    #setup experiment sub directories\n    experiments = [\"DMS_MaP\",\"2A3_MaP\",\"test\"]\n    for experiment in experiments:\n        exp_dir = os.path.join(TEMP_DIR,experiment)\n        if not os.path.isdir(exp_dir):\n            os.mkdir(exp_dir)\n\n    os.listdir(TEMP_DIR)\n    \n\ndef gen_path_name(seq_id, experiment):\n    return os.path.join(TEMP_DIR,experiment,f\"{seq_id}.png\")\n\ndef pad_sequence(sequence, max_length=457, padding_value=\"Z\"):\n    # Calculate how many padding values are needed\n    padding_length = max_length - len(sequence)\n    \n    # Create the padded sequence\n    padded_sequence = list(sequence) + [padding_value] * padding_length\n    \n    return padded_sequence\n\ndef pad_reactivity(sequence, max_length=457, padding_value=0.0):\n    # Calculate how many padding values are needed\n    padding_length = max_length - len(sequence)\n    \n    # Create the padded sequence\n    padded_sequence = sequence + [padding_value] * padding_length\n    \n    return padded_sequence\n\ndef encode_nuc(nuc):\n    nuc_map = {\n        \"A\": [1,0,0,0,0],\n        \"C\": [0,1,0,0,0],\n        \"G\": [0,0,1,0,0],\n        \"U\": [0,0,0,1,0],\n        \"Z\": [0,0,0,0,1]\n    }\n    \n    return nuc_map[nuc]\n\n\ndef encode_sequence(sequence):\n    return [encode_nuc(s) for s in sequence]\n\n\ndef process_experiment_data(experiment_type):\n    print(\"Processing Data...\")\n\n    train_load = pd.read_csv(f\"./{experiment_type}.csv\")\n    experiment_data = train_load.drop(columns=train_load.filter(like='error').columns,axis=1)\n    \n    del train_load\n    gc.collect()    \n\n    experiment_data = experiment_data.drop(['reads', 'SN_filter','signal_to_noise'],axis=1)\n    reactivity_cols = experiment_data.filter(like='reactivity').columns\n    \n    experiment_data[reactivity_cols] = experiment_data[reactivity_cols]\n    experiment_data['reactivity'] = experiment_data[reactivity_cols].apply(lambda row: row.fillna(row.mean()), axis=1).fillna(0.0).clip(lower=0.0,upper=1.0).values.tolist()\n    experiment_data = experiment_data.drop(columns=experiment_data.filter(like='reactivity_').columns,axis=1)\n\n    experiment_data = experiment_data.drop(['dataset_name'],axis=1)\n    experiment_data[\"filename\"] = experiment_data[\"sequence_id\"].apply(gen_path_name,args=(experiment_type,))\n\n    \n    print(\"Data Processed!\")\n    \n    return experiment_data\n    \n\ndef generate_images(experiment_data):\n    print(\"Generating Sequence Images\")\n    sequences = experiment_data[[\"sequence\",\"filename\"]]\n    sequences = sequences.values.tolist()\n    \n    for sequence, path in sequences:\n        sequence = pad_sequence(sequence)\n        encoded = encode_sequence(sequence)\n        encoded = np.array(encoded).astype(np.uint8)\n        im = Image.fromarray(encoded)\n        im.save(path)\n        \n        del sequence\n\n    plt.imshow(encoded)\n    plt.xlim((0,5))\n    plt.ylim((0,10))\n\n    del sequences, encoded\n    gc.collect()\n    \n    print(\"Images Generated\")\n\n\ndef prepare_training_data(experiment_data):\n    experiment_data_train = experiment_data[[\"filename\",\"reactivity\"]]\n    \n    cols = [f\"reactivity_{i}\" for i in range(457)]\n    experiment_data_train['reactivity'] = experiment_data_train['reactivity'].apply(pad_reactivity)\n    experiment_data_train[cols] = pd.DataFrame(experiment_data_train.reactivity.tolist(), index= experiment_data_train.index)\n    experiment_data_train = experiment_data_train.drop([\"reactivity\"],axis=1)\n\n    del experiment_data\n    gc.collect()\n    \n    return experiment_data_train, cols\n\n\ndef train_test_val_split(experiment_data_train, train_frac, test_frac):\n    \n    train=experiment_data_train.sample(frac=train_frac,random_state=777)\n    test_val=experiment_data_train.drop(train.index)\n    test = test_val.sample(frac=test_frac,random_state=777)\n    valid= test_val.drop(test.index)\n    \n    del experiment_data_train\n    gc.collect()\n    \n    return train, test, valid\n\ndef build_image_generators(train, test, valid):\n    datagen=tf.keras.preprocessing.image.ImageDataGenerator()\n\n    train_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=train,\n            x_col='filename',\n            y_col=cols,\n            class_mode='raw',\n            batch_size=BATCH_SIZE,\n            target_size=(457, 5),\n            shuffle=True\n    )\n\n    valid_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=valid,\n            x_col='filename',\n            y_col=cols,\n            class_mode='raw',\n            batch_size=BATCH_SIZE,\n            target_size=(457, 5),\n            shuffle=True\n    )\n\n    test_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=test,\n            x_col='filename',\n            y_col=cols,\n            target_size=(457, 5),\n            batch_size=BATCH_SIZE,\n            seed=42,\n            shuffle=False,\n            class_mode=None\n            )\n\n    del train\n    del valid\n    gc.collect()\n    \n    return train_generator, test_generator, valid_generator\n\ndef get_model():\n    \n    model = tf.keras.Sequential([\n        tf.keras.layers.Conv2D(2,(2,2), activation='relu'),\n        tf.keras.layers.MaxPooling2D(pool_size=(2,2)),\n\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='relu'),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='relu'),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='sigmoid')\n        \n    ])\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n            loss=tf.keras.losses.MeanSquaredError(),\n            )\n\n    return model\n\ndef fit_model(train_generator, valid_generator):\n    try:\n        del model\n        gc.collect()\n    except:\n        print(\"No Model Found\")\n\n    with tf.device('gpu:0'):\n\n        model = get_model()\n        history = model.fit(train_generator,validation_data=valid_generator, epochs=EPOCHS)\n        \n    \n    return model, history\n\ndef plot_results(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.legend([\"Loss\",\"Val Loss\"])\n","metadata":{"execution":{"iopub.status.busy":"2023-11-08T23:26:57.153077Z","iopub.execute_input":"2023-11-08T23:26:57.153446Z","iopub.status.idle":"2023-11-08T23:26:57.184999Z","shell.execute_reply.started":"2023-11-08T23:26:57.153418Z","shell.execute_reply":"2023-11-08T23:26:57.184005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN:\n    files = os.listdir('./')\n    if not \"2A3_MaP.csv\" in files or not \"DMS_MaP.csv\" in files: \n        split_dataset()\n\n    experiment_2a3 = \"2A3_MaP\"\n    setup_directories()\n    experiment_data = process_experiment_data(experiment_2a3)\n    if  len(os.listdir(os.path.join(TEMP_DIR,experiment_2a3))) < experiment_data.shape[0]:\n        generate_images(experiment_data)\n\n    experiment_data_train, cols = prepare_training_data(experiment_data)\n    train, test, valid = train_test_val_split(experiment_data_train, 0.7, 0.66)\n    train_generator, test_generator, valid_generator = build_image_generators(train, test, valid)\n\n\n\n\n    model_2a3, history = fit_model(train_generator, valid_generator)\n\n    if PLOT_RESULTS:\n        plot_results(history)\n    del history\n    gc.collect()\n\n    if TESTING:\n        test_generator.reset()\n        preds = model_2a3.predict(test_generator,batch_size=BATCH_SIZE)\n        test_vals = test[cols]\n        test_vals = test_vals.values.tolist()\n        print(f\"Model {experiment_2a3}, MSE:  {mean_squared_error(preds,test_vals)}\")\n\n    del train_generator, valid_generator, test_generator\n    del preds, test_vals, test\n    gc.collect()\n\n    experiment_dms = \"DMS_MaP\"\n    setup_directories()\n    experiment_data = process_experiment_data(experiment_dms)\n    if  len(os.listdir(os.path.join(TEMP_DIR,experiment_dms))) < experiment_data.shape[0]:\n        generate_images(experiment_data)\n\n    experiment_data_train, cols = prepare_training_data(experiment_data)\n    train, test, valid = train_test_val_split(experiment_data_train, 0.7, 0.66)\n    train_generator, test_generator, valid_generator = build_image_generators(train, test, valid)\n\n\n    model_dms, history = fit_model(train_generator, valid_generator)\n\n    if PLOT_RESULTS:\n        plot_results(history)\n    del history\n\n    if TESTING:\n        test_generator.reset()\n        preds = model_dms.predict(test_generator,batch_size=BATCH_SIZE)\n        test_vals = test[cols]\n        test_vals = test_vals.values.tolist()\n        print(f\"Model {experiment_dms}, MSE:  {mean_squared_error(preds,test_vals)}\")\n\n    del train_generator, valid_generator, test_generator\n    del preds, test_vals, test\n    gc.collect()\n\n    model_dms.save_weights('/kaggle/working/checkpoints/dms_model')\n    model_2a3.save_weights('/kaggle/working/checkpoints/2a3_model')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-08T23:34:01.284258Z","iopub.execute_input":"2023-11-08T23:34:01.284642Z","iopub.status.idle":"2023-11-08T23:57:20.632794Z","shell.execute_reply.started":"2023-11-08T23:34:01.284612Z","shell.execute_reply":"2023-11-08T23:57:20.631670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"os.listdir(\"","metadata":{}}]}