{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":51294,"databundleVersionId":6923401,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install arnie","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-27T08:06:35.969079Z","iopub.execute_input":"2023-11-27T08:06:35.969480Z","iopub.status.idle":"2023-11-27T08:06:48.708991Z","shell.execute_reply.started":"2023-11-27T08:06:35.969449Z","shell.execute_reply":"2023-11-27T08:06:48.707790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!conda config --set auto_update_conda false","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:06:48.711620Z","iopub.execute_input":"2023-11-27T08:06:48.712416Z","iopub.status.idle":"2023-11-27T08:06:50.246142Z","shell.execute_reply.started":"2023-11-27T08:06:48.712375Z","shell.execute_reply":"2023-11-27T08:06:50.244673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!conda config --show-sources","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:06:50.248169Z","iopub.execute_input":"2023-11-27T08:06:50.248829Z","iopub.status.idle":"2023-11-27T08:06:51.751611Z","shell.execute_reply.started":"2023-11-27T08:06:50.248794Z","shell.execute_reply":"2023-11-27T08:06:51.750555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r /tmp/conda/","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:06:51.754981Z","iopub.execute_input":"2023-11-27T08:06:51.755729Z","iopub.status.idle":"2023-11-27T08:06:52.764868Z","shell.execute_reply.started":"2023-11-27T08:06:51.755670Z","shell.execute_reply":"2023-11-27T08:06:52.763421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!conda install -c bioconda contrafold --yes","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:06:52.766509Z","iopub.execute_input":"2023-11-27T08:06:52.766906Z","iopub.status.idle":"2023-11-27T08:08:25.666426Z","shell.execute_reply.started":"2023-11-27T08:06:52.766864Z","shell.execute_reply":"2023-11-27T08:08:25.665082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nimport sys\n\n%env CONTRAFOLD_PATH=/opt/conda/bin/contrafold\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:08:25.668129Z","iopub.execute_input":"2023-11-27T08:08:25.668549Z","iopub.status.idle":"2023-11-27T08:08:25.678296Z","shell.execute_reply.started":"2023-11-27T08:08:25.668514Z","shell.execute_reply":"2023-11-27T08:08:25.677301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\n\nimport numpy as np  # linear algebra\nimport pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\n\nwarnings.filterwarnings(\"ignore\")\n\n\nimport time\nfrom multiprocessing import Pool\n\nimport arnie.utils as utils\nimport numpy as np\nimport pandas as pd\nfrom arnie.bpps import bpps\nfrom arnie.free_energy import free_energy\nfrom arnie.mea.mea import MEA\nfrom arnie.mfe import mfe\nfrom arnie.pfunc import pfunc\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:08:25.679549Z","iopub.execute_input":"2023-11-27T08:08:25.679938Z","iopub.status.idle":"2023-11-27T08:08:26.171332Z","shell.execute_reply.started":"2023-11-27T08:08:25.679909Z","shell.execute_reply":"2023-11-27T08:08:26.169975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"utils.print_path_files()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:08:26.173179Z","iopub.execute_input":"2023-11-27T08:08:26.174121Z","iopub.status.idle":"2023-11-27T08:08:26.180733Z","shell.execute_reply.started":"2023-11-27T08:08:26.174058Z","shell.execute_reply":"2023-11-27T08:08:26.179622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def proc1(arg):\n    sequence = arg[0]\n    id = arg[1]\n    bp_matrix = bpps(sequence, package=\"contrafold\")\n    return (\n        id,\n        sequence,\n        bp_matrix,\n    )","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:08:26.182470Z","iopub.execute_input":"2023-11-27T08:08:26.183190Z","iopub.status.idle":"2023-11-27T08:08:26.245322Z","shell.execute_reply.started":"2023-11-27T08:08:26.183147Z","shell.execute_reply":"2023-11-27T08:08:26.243096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(\n    \"/kaggle/input/stanford-ribonanza-rna-folding/train_data.csv\"\n)\n        ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:08:26.248276Z","iopub.execute_input":"2023-11-27T08:08:26.248887Z","iopub.status.idle":"2023-11-27T08:10:06.665423Z","shell.execute_reply.started":"2023-11-27T08:08:26.248851Z","shell.execute_reply":"2023-11-27T08:10:06.664178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_THRE = 40\nCHUNK_NUM = 10000\nchunk_start = 0\n# DATALEN = len(train_data) // 2\nDATALEN = 300000\ntrain_data = train_data[train_data.experiment_type == \"2A3_MaP\"].reset_index(drop=True)\nnew_df = pd.DataFrame()\nbp_matrix_dir = \"bp_matrix\"\nos.makedirs(bp_matrix_dir, exist_ok=True)\np = Pool(processes=MAX_THRE)\nwhile chunk_start < (DATALEN):\n    os.makedirs(\n        os.path.join(bp_matrix_dir, f\"train_{chunk_start}_{chunk_start + CHUNK_NUM}\"),\n        exist_ok=True,\n    )\n    print(\n        f\"Start process: {chunk_start} - {chunk_start + CHUNK_NUM} / {len(train_data)}\"\n    )\n    train_data_small = train_data.iloc[chunk_start : chunk_start + CHUNK_NUM]\n    target_df = train_data_small[[\"sequence_id\", \"sequence\"]]\n    total_items = len(target_df)\n    li = []\n    for i, arr in enumerate(target_df[[\"sequence\", \"sequence_id\"]].values):\n        li.append([arr[0], arr[1]])\n    results = []\n    start_time = time.time()  # Start tracking time\n    for i, ret in enumerate(p.imap(proc1, li)):\n        results.append(ret)\n        elapsed_time = time.time() - start_time\n        estimated_total_time = (elapsed_time / (i + 1)) * total_items\n        remaining_time = estimated_total_time - elapsed_time\n        print(\n            f\"\\rProcessed item {i + 1}/{total_items}. Elapsed time: {elapsed_time:.2f}s. Estimated remaining time: {remaining_time:.2f}s.\",\n            end=\"\",\n        )\n    df = pd.DataFrame(\n        results,\n        columns=[\n            \"sequence_id\",\n            \"sequence\",\n            \"bp_matrix\",\n        ],\n    )\n    for sequence_id in tqdm(target_df[\"sequence_id\"].unique()):\n        np.save(\n            os.path.join(\n                bp_matrix_dir,\n                f\"train_{chunk_start}_{chunk_start + CHUNK_NUM}\",\n                sequence_id,\n            ),\n            df.loc[:, [\"bp_matrix\"]].iloc[0].to_numpy()[0].astype(np.float32),\n        )\n    !zip -r --quiet /kaggle/working/bp_matrix/\"train_{chunk_start}_{chunk_start + CHUNK_NUM}\".zip /kaggle/working/bp_matrix/\"train_{chunk_start}_{chunk_start + CHUNK_NUM}\"\n    !rm -r /kaggle/working/bp_matrix/\"train_{chunk_start}_{chunk_start + CHUNK_NUM}\"\n    chunk_start += CHUNK_NUM\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T08:12:01.272234Z","iopub.execute_input":"2023-11-27T08:12:01.272762Z","iopub.status.idle":"2023-11-27T08:14:47.660510Z","shell.execute_reply.started":"2023-11-27T08:12:01.272716Z","shell.execute_reply":"2023-11-27T08:14:47.644402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}