{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport gc\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom sklearn.metrics import mean_squared_error\n\n# To ignore warinings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# constants \nTEMP_DIR = \"/kaggle/temp\"\nTESTING = True\nPLOT_RESULTS = True\nTRAIN = False\nEVAL = True\n\n\n# Model Hyperparameters\nEPOCHS = 1\nBATCH_SIZE = 32\nLEARNING_RATE = 1e-3\n\n\nos.listdir(\"/kaggle/input/rna-cnns-v1\")","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:34:08.975278Z","iopub.execute_input":"2023-11-08T15:34:08.975654Z","iopub.status.idle":"2023-11-08T15:34:16.292725Z","shell.execute_reply.started":"2023-11-08T15:34:08.975626Z","shell.execute_reply":"2023-11-08T15:34:16.291514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport os\n\n\ndef split_dataset():\n    dataset = pd.read_csv(\"/kaggle/input/stanford-ribonanza-rna-folding/train_data.csv\")\n    experiments = [\"DMS_MaP\",\"2A3_MaP\"]\n\n    for experiment in experiments:\n        experiment_data = dataset[dataset['experiment_type'] == experiment]\n        experiment_data.to_csv(f\"./{experiment}.csv\",index=False)\n    \n        del experiment_data\n    del dataset\n    del experiments\n    gc.collect()\n\ndef setup_directories():\n    \n    # setup temporary directory \n    if not os.path.isdir(TEMP_DIR):\n        os.mkdir(TEMP_DIR)\n    os.listdir(\"/kaggle\")\n\n    #setup experiment sub directories\n    experiments = [\"DMS_MaP\",\"2A3_MaP\",\"test\"]\n    for experiment in experiments:\n        exp_dir = os.path.join(TEMP_DIR,experiment)\n        if not os.path.isdir(exp_dir):\n            os.mkdir(exp_dir)\n\n    os.listdir(TEMP_DIR)\n    \n\ndef gen_path_name(seq_id, experiment):\n    return os.path.join(TEMP_DIR,experiment,f\"{seq_id}.png\")\n\ndef pad_sequence(sequence, max_length=457, padding_value=\"Z\"):\n    # Calculate how many padding values are needed\n    padding_length = max_length - len(sequence)\n    \n    # Create the padded sequence\n    padded_sequence = list(sequence) + [padding_value] * padding_length\n    \n    return padded_sequence\n\ndef pad_reactivity(sequence, max_length=457, padding_value=0):\n    # Calculate how many padding values are needed\n    padding_length = max_length - len(sequence)\n    \n    # Create the padded sequence\n    padded_sequence = sequence + [padding_value] * padding_length\n    \n    return padded_sequence\n\ndef encode_nuc(nuc):\n    nuc_map = {\n        \"A\": [1,0,0,0,0],\n        \"C\": [0,1,0,0,0],\n        \"G\": [0,0,1,0,0],\n        \"U\": [0,0,0,1,0],\n        \"Z\": [0,0,0,0,1]\n    }\n    \n    return nuc_map[nuc]\n\n\ndef encode_sequence(sequence):\n    return [encode_nuc(s) for s in sequence]\n\n\ndef process_experiment_data(experiment_type):\n    print(\"Processing Data...\")\n\n    train_load = pd.read_csv(f\"./{experiment_type}.csv\")\n    experiment_data = train_load.drop(columns=train_load.filter(like='error').columns,axis=1)\n    \n    del train_load\n    gc.collect()    \n\n    experiment_data = experiment_data.drop(['reads', 'SN_filter','signal_to_noise'],axis=1)\n    reactivity_cols = experiment_data.filter(like='reactivity').columns\n    experiment_data = experiment_data.fillna(0)\n    experiment_data[reactivity_cols] = experiment_data[reactivity_cols].clip(lower=0,upper=1)\n    experiment_data['reactivity'] = experiment_data[reactivity_cols].values.tolist()\n    experiment_data = experiment_data.drop(columns=experiment_data.filter(like='reactivity_').columns,axis=1)\n\n    experiment_data = experiment_data.drop(['dataset_name'],axis=1)\n    experiment_data[\"filename\"] = experiment_data[\"sequence_id\"].apply(gen_path_name,args=(experiment_type,))\n\n    \n    print(\"Data Processed!\")\n    \n    return experiment_data\n    \n\ndef generate_images(experiment_data):\n    print(\"Generating Sequence Images\")\n    sequences = experiment_data[[\"sequence\",\"filename\"]]\n    sequences = sequences.values.tolist()\n    \n    for sequence, path in sequences:\n        sequence = pad_sequence(sequence)\n        encoded = encode_sequence(sequence)\n        encoded = np.array(encoded).astype(np.uint8)\n        im = Image.fromarray(encoded)\n        im.save(path)\n        \n        del sequence\n\n    plt.imshow(encoded)\n    plt.xlim((0,5))\n    plt.ylim((0,10))\n\n    del sequences, encoded\n    gc.collect()\n    \n    print(\"Images Generated\")\n\n\ndef prepare_training_data(experiment_data):\n    experiment_data_train = experiment_data[[\"filename\",\"reactivity\"]]\n    \n    cols = [f\"reactivity_{i}\" for i in range(457)]\n    experiment_data_train['reactivity'] = experiment_data_train['reactivity'].apply(pad_reactivity)\n    experiment_data_train[cols] = pd.DataFrame(experiment_data_train.reactivity.tolist(), index= experiment_data_train.index)\n    experiment_data_train = experiment_data_train.drop([\"reactivity\"],axis=1)\n\n    del experiment_data\n    gc.collect()\n    \n    return experiment_data_train, cols\n\n\ndef train_test_val_split(experiment_data_train, train_frac, test_frac):\n    \n    train=experiment_data_train.sample(frac=train_frac,random_state=777)\n    test_val=experiment_data_train.drop(train.index)\n    test = test_val.sample(frac=test_frac,random_state=777)\n    valid= test_val.drop(test.index)\n    \n    del experiment_data_train\n    gc.collect()\n    \n    return train, test, valid\n\ndef build_image_generators(train, test, valid):\n    datagen=tf.keras.preprocessing.image.ImageDataGenerator()\n\n    train_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=train,\n            x_col='filename',\n            y_col=cols,\n            class_mode='raw',\n            batch_size=BATCH_SIZE,\n            target_size=(457, 5),\n            shuffle=True\n    )\n\n    valid_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=valid,\n            x_col='filename',\n            y_col=cols,\n            class_mode='raw',\n            batch_size=BATCH_SIZE,\n            target_size=(457, 5),\n            shuffle=True\n    )\n\n    test_generator = datagen.flow_from_dataframe(\n            directory=None,\n            dataframe=test,\n            x_col='filename',\n            y_col=cols,\n            target_size=(457, 5),\n            batch_size=BATCH_SIZE,\n            seed=42,\n            shuffle=False,\n            class_mode=None\n            )\n\n    del train\n    del valid\n    gc.collect()\n    \n    return train_generator, test_generator, valid_generator\n\ndef get_model():\n    \n    model = tf.keras.Sequential([\n        tf.keras.layers.Conv2D(2,(2,2), activation='relu'),\n        tf.keras.layers.MaxPooling2D(pool_size=(2,2)),\n\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='relu'),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='relu'),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(457, activation='sigmoid')\n        \n    ])\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n            loss=tf.keras.losses.MeanSquaredError(),\n            )\n\n    return model\n\ndef fit_model(train_generator, valid_generator):\n    try:\n        del model\n        gc.collect()\n    except:\n        print(\"No Model Found\")\n\n    with tf.device('gpu:0'):\n\n        model = get_model()\n        history = model.fit(train_generator,validation_data=valid_generator, epochs=EPOCHS)\n        \n    \n    return model, history\n\ndef plot_results(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.legend([\"Loss\",\"Val Loss\"])\n\n\ndef pad_sequence(sequence, max_length=457, padding_value=\"Z\"):\n    # Calculate how many padding values are needed\n    padding_length = max_length - len(sequence)\n    \n    # Create the padded sequence\n    padded_sequence = list(sequence) + [padding_value] * padding_length\n    \n    return padded_sequence\n\ndef length(l):\n    return len(l)\n\ndef dummy_labels(df):\n    cols = [f\"reactivity_{i}\" for i in range(457)]\n    labels = [[0 for i in range(457)] for i in range(df.shape[0])]\n    df[cols] = pd.DataFrame(labels, index= df.index)\n    \n    return df, cols\n\n\ndef setup_directories():\n    \n    # setup temporary directory \n    if not os.path.isdir(TEMP_DIR):\n        os.mkdir(TEMP_DIR)\n    os.listdir(\"/kaggle\")\n\n    #setup experiment sub directories\n    experiments = [\"DMS_MaP\",\"2A3_MaP\",\"test\"]\n    for experiment in experiments:\n        exp_dir = os.path.join(TEMP_DIR,experiment)\n        if not os.path.isdir(exp_dir):\n            os.mkdir(exp_dir)\n\n    os.listdir(TEMP_DIR)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:34:16.294981Z","iopub.execute_input":"2023-11-08T15:34:16.295752Z","iopub.status.idle":"2023-11-08T15:34:16.562131Z","shell.execute_reply.started":"2023-11-08T15:34:16.295711Z","shell.execute_reply":"2023-11-08T15:34:16.561022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if EVAL:\n    eval_samples = pd.read_csv(\"/kaggle/input/stanford-ribonanza-rna-folding/test_sequences.csv\")\n    gc.collect()\n\n    setup_directories()\n    eval_samples['sequence'] = eval_samples[\"sequence\"].apply(pad_sequence)\n    gc.collect()\n\n\n    eval_samples, cols = dummy_labels(eval_samples)\n\n    eval_samples.head()\n\n    # import tensorflow as tf\n\n\n\n    eval_samples['filename'] = eval_samples[\"sequence_id\"].apply(gen_path_name,args=(\"test\",))\n    generate_images(eval_samples)\n\n    datagen=tf.keras.preprocessing.image.ImageDataGenerator()\n\n    test_generator = datagen.flow_from_dataframe(\n                directory=None,\n                dataframe=eval_samples,\n                x_col='filename',\n                y_col=cols,\n                target_size=(457, 5),\n                batch_size=32,\n                seed=42,\n                shuffle=False,\n                class_mode=None\n                )\n\n    print(test_generator)\n    \n","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:34:27.178256Z","iopub.execute_input":"2023-11-08T15:34:27.179124Z","iopub.status.idle":"2023-11-08T16:05:45.438919Z","shell.execute_reply.started":"2023-11-08T15:34:27.179074Z","shell.execute_reply":"2023-11-08T16:05:45.437648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_samples.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:05:45.441818Z","iopub.execute_input":"2023-11-08T16:05:45.442187Z","iopub.status.idle":"2023-11-08T16:05:45.476543Z","shell.execute_reply.started":"2023-11-08T16:05:45.442127Z","shell.execute_reply":"2023-11-08T16:05:45.475493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2a3 = get_model()\nmodel_2a3.load_weights('/kaggle/input/rna-cnns-v1/checkpoints/2a3_model')\n\nmodel_dms = get_model()\nmodel_dms.load_weights('/kaggle/input/rna-cnns-v1/checkpoints/dms_model')\n\nprint(model_dms)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:05:45.477899Z","iopub.execute_input":"2023-11-08T16:05:45.478307Z","iopub.status.idle":"2023-11-08T16:05:46.070362Z","shell.execute_reply.started":"2023-11-08T16:05:45.478268Z","shell.execute_reply":"2023-11-08T16:05:46.069344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator.reset()\npreds_2a3 = model_2a3.predict(test_generator)\ntest_generator.reset()\npreds_dms= model_dms.predict(test_generator)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:05:46.072423Z","iopub.execute_input":"2023-11-08T16:05:46.072803Z","iopub.status.idle":"2023-11-08T16:31:15.525183Z","shell.execute_reply.started":"2023-11-08T16:05:46.072771Z","shell.execute_reply":"2023-11-08T16:31:15.523638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_samples = eval_samples[[\"id_min\",\"id_max\"]]","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:31:15.539183Z","iopub.execute_input":"2023-11-08T16:31:15.540083Z","iopub.status.idle":"2023-11-08T16:31:15.567134Z","shell.execute_reply.started":"2023-11-08T16:31:15.540036Z","shell.execute_reply":"2023-11-08T16:31:15.566238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_samples.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:31:16.284814Z","iopub.execute_input":"2023-11-08T16:31:16.286342Z","iopub.status.idle":"2023-11-08T16:31:16.309136Z","shell.execute_reply.started":"2023-11-08T16:31:16.286253Z","shell.execute_reply":"2023-11-08T16:31:16.306841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_samples['n'] = eval_samples['id_max'] - eval_samples['id_min']\neval_samples = eval_samples[\"n\"]","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:36:57.339928Z","iopub.execute_input":"2023-11-08T16:36:57.340752Z","iopub.status.idle":"2023-11-08T16:36:57.360600Z","shell.execute_reply.started":"2023-11-08T16:36:57.340709Z","shell.execute_reply":"2023-11-08T16:36:57.359739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_samples = eval_samples.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:38:23.342921Z","iopub.execute_input":"2023-11-08T16:38:23.343381Z","iopub.status.idle":"2023-11-08T16:38:23.367309Z","shell.execute_reply.started":"2023-11-08T16:38:23.343343Z","shell.execute_reply":"2023-11-08T16:38:23.366329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, ind in enumerate(eval_samples):\n    np.put(preds_2a3[i], list(range(ind+1,457)),[np.nan])\n    np.put(preds_dms[i], list(range(ind+1,457)),[np.nan])","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:40:47.590888Z","iopub.execute_input":"2023-11-08T16:40:47.591383Z","iopub.status.idle":"2023-11-08T16:41:53.729417Z","shell.execute_reply.started":"2023-11-08T16:40:47.591351Z","shell.execute_reply":"2023-11-08T16:41:53.728101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_2a3 = preds_2a3.flatten()\npreds_2a3 = preds_2a3[~np.isnan(preds_2a3)]\n\npreds_dms = preds_dms.flatten()\npreds_dms =preds_dms[~np.isnan(preds_dms)]","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:45:04.888100Z","iopub.execute_input":"2023-11-08T16:45:04.888552Z","iopub.status.idle":"2023-11-08T16:45:10.197420Z","shell.execute_reply.started":"2023-11-08T16:45:04.888517Z","shell.execute_reply":"2023-11-08T16:45:10.196067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_generator\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:45:30.253884Z","iopub.execute_input":"2023-11-08T16:45:30.255134Z","iopub.status.idle":"2023-11-08T16:45:35.884236Z","shell.execute_reply.started":"2023-11-08T16:45:30.255070Z","shell.execute_reply":"2023-11-08T16:45:35.883165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_dms.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-08T16:45:52.791720Z","iopub.execute_input":"2023-11-08T16:45:52.792108Z","iopub.status.idle":"2023-11-08T16:45:52.799198Z","shell.execute_reply.started":"2023-11-08T16:45:52.792079Z","shell.execute_reply":"2023-11-08T16:45:52.798071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = np.arange(preds_dms.shape[0], dtype=np.int64)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T05:19:15.030130Z","iopub.status.idle":"2023-11-08T05:19:15.030571Z","shell.execute_reply.started":"2023-11-08T05:19:15.030355Z","shell.execute_reply":"2023-11-08T05:19:15.030376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nimport polars as pl\nschema={k:pl.Float32 for k in ['id', 'reactivity_DMS_MaP', 'reactivity_2A3_MaP']}\nschema['id'] = pl.Int64\nsub_path = './submission.csv'\ndf = pl.DataFrame(\n    data=[ids, preds_dms, preds_2a3],\n    schema=schema\n)\ndf.write_csv(sub_path, float_precision=4) \nprint(df.shape)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-08T05:19:15.035861Z","iopub.status.idle":"2023-11-08T05:19:15.036539Z","shell.execute_reply.started":"2023-11-08T05:19:15.036210Z","shell.execute_reply":"2023-11-08T05:19:15.036243Z"},"trusted":true},"execution_count":null,"outputs":[]}]}