{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":87793,"databundleVersionId":12024591,"sourceType":"competition"},{"sourceId":238022034,"sourceType":"kernelVersion"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-06T16:39:46.399589Z","iopub.execute_input":"2025-05-06T16:39:46.399972Z","iopub.status.idle":"2025-05-06T16:39:50.065299Z","shell.execute_reply.started":"2025-05-06T16:39:46.399939Z","shell.execute_reply":"2025-05-06T16:39:50.063887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seq_df = pd.read_csv('/kaggle/input/feature-extraction-global/seq_df_global.csv')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-06T16:39:50.067454Z","iopub.execute_input":"2025-05-06T16:39:50.068010Z","iopub.status.idle":"2025-05-06T16:39:50.555554Z","shell.execute_reply.started":"2025-05-06T16:39:50.067977Z","shell.execute_reply":"2025-05-06T16:39:50.554503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\ncsv_path = '/kaggle/input/feature-extraction-global/seq_df_global.csv'\ndf = pd.read_csv(csv_path)\n\none_d_cols = [\n    \"onehot\",\"pair_flag\",\"loop_type_onehot\",\n    \"dG_loop_res\",\"sa\",\"flexibility_index\"\n]\ntwo_d_cols = [\n    \"pp\",\"mi_matrix\",\"dca\",\"covariance\",\n    \"bp_potentials\",\"gt_distances\",\"contact_map\",\"angle_matrix\",\"motif_match_matrix\"\n]\n\ndef parse_array_string(x):\n    if not isinstance(x, str):\n        return np.array(x, dtype=float)\n    rows = re.findall(r'\\[([^\\[\\]]*\\d+[^\\[\\]]*)\\]', x)\n    mat = []\n    for r in rows:\n        nums = re.findall(r'-?\\d+\\.?\\d*(?:[eE][-+]?\\d+)?', r)\n        mat.append([float(n) for n in nums])\n    if not mat:\n        return np.empty((0,0), dtype=float)\n    ml = max(len(r) for r in mat)\n    for r in mat:\n        r.extend([0.0] * (ml - len(r)))\n    return np.array(mat, dtype=float)\n\nfor c in one_d_cols + two_d_cols:\n    df[c] = df[c].apply(parse_array_string)\n\nglobal_numeric = [\"freq_A\",\"freq_C\",\"freq_G\",\"freq_U\",\n                  \"total_dG\",\"msa_depth\",\"msa_diversity\",\n                  \"gc_content\",\"n_stems\",\"n_loops\",\"partner_chains_count\"]\nglobal_binary  = [\"has_metal\",\"has_ligand\"]\nglobal_cat     = [\"ligand_metal_category\",\"experimental_method\"]\ndf_cat    = pd.get_dummies(df[global_cat].astype(str), prefix=global_cat)\ndf_global = pd.concat([df[global_numeric+global_binary], df_cat], axis=1)\nglobal_cols = df_global.columns.tolist()\n\nclass RNADataset(Dataset):\n    def __init__(self, df, one_d_cols, two_d_cols, global_cols):\n        self.df          = df.reset_index(drop=True)\n        self.one_d_cols  = one_d_cols\n        self.two_d_cols  = two_d_cols\n        self.global_cols = global_cols\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        onehot = np.asarray(row[self.one_d_cols[0]], dtype=float)\n        L = onehot.shape[0]\n\n        parts = []\n        for c in self.one_d_cols:\n            arr = np.asarray(row[c], dtype=float)\n            arr = np.atleast_2d(arr)\n            if arr.shape[0] == 1 and arr.shape[1] == L:\n                arr = arr.T\n            h, k = arr.shape\n            if h != L:\n                new = np.zeros((L, k), dtype=float)\n                mh = min(h, L)\n                new[:mh, :] = arr[:mh, :]\n                arr = new\n            parts.append(torch.from_numpy(arr).float())\n        X1 = torch.cat(parts, dim=1)\n\n        mats = []\n        for c in self.two_d_cols:\n            mat = np.asarray(row[c], dtype=float)\n            mat = np.atleast_2d(mat)\n            h, w = mat.shape\n            if (h, w) != (L, L):\n                new = np.zeros((L, L), dtype=float)\n                mh = min(h, L)\n                mw = min(w, L)\n                new[:mh, :mw] = mat[:mh, :mw]\n                mat = new\n            mats.append(torch.from_numpy(mat).float().unsqueeze(-1))\n        X2 = torch.cat(mats, dim=-1)\n        \n        G = torch.from_numpy(df_global.iloc[idx].to_numpy(dtype=float))\n\n        return X1, X2, G, L\n\ndef collate_fn(batch):\n    lengths = [b[3] for b in batch]\n    Lmax    = max(lengths)\n    B       = len(batch)\n    F1      = batch[0][0].shape[1]\n    F2      = batch[0][1].shape[2]\n    Gdim    = batch[0][2].shape[0]\n\n    X1b = torch.zeros((B, Lmax, F1), dtype=torch.float32)\n    X2b = torch.zeros((B, Lmax, Lmax, F2), dtype=torch.float32)\n    Gb  = torch.zeros((B, Gdim),     dtype=torch.float32)\n\n    for i, (X1, X2, G, L) in enumerate(batch):\n        X1b[i, :L, :]     = X1.half()\n        X2b[i, :L, :L, :] = X2.half()\n        Gb[i]             = G.half()\n\n    return X1b, X2b, Gb, torch.tensor(lengths)\n\ndataset = RNADataset(df, one_d_cols, two_d_cols, global_cols)\nloader  = DataLoader(\n    dataset,\n    batch_size=8,          \n    shuffle=False,\n    num_workers=0,         \n    collate_fn=collate_fn,\n    pin_memory=True\n)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nfor X1b, X2b, Gb, lengths in loader:\n    X1b, X2b, Gb = X1b.to(device), X2b.to(device), Gb.to(device)\n    print(\"X1:\", X1b.shape, X1b.dtype, X1b.device)\n    print(\"X2:\", X2b.shape, X2b.dtype, X2b.device)\n    print(\"G: \",  Gb.shape,  Gb.dtype,  Gb.device)\n    print(\"lengths:\", lengths)\n    break\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-06T16:39:56.546208Z","iopub.execute_input":"2025-05-06T16:39:56.547055Z","iopub.status.idle":"2025-05-06T16:40:03.222353Z","shell.execute_reply.started":"2025-05-06T16:39:56.547020Z","shell.execute_reply":"2025-05-06T16:40:03.220849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nbatch_idx = 0\nseq_idx   = df.index[batch_idx]\nL = lengths[batch_idx].item()\n\nprint(f\"Examining sequence {df.loc[seq_idx,'target_id']} (length {L})\\n\")\n\nfor chan, name in enumerate(two_d_cols):\n    raw = np.asarray(df.loc[seq_idx, name], dtype=float)\n    bat = X2b[batch_idx, :L, :L, chan].cpu().numpy()\n\n    print(f\"--- {name} ---\")\n    print(f\" raw shape: {raw.shape}, tensor shape: {bat.shape}\")\n    print(f\" raw[0:3,0:3]:\\n{raw[:3,:3]}\")\n    print(f\" bat[0:3,0:3]:\\n{bat[:3,:3]}\")\n    print()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-06T16:41:41.961418Z","iopub.execute_input":"2025-05-06T16:41:41.961792Z","iopub.status.idle":"2025-05-06T16:41:41.974982Z","shell.execute_reply.started":"2025-05-06T16:41:41.961764Z","shell.execute_reply":"2025-05-06T16:41:41.973955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pickle\nimport torch\n\nos.makedirs(\"/kaggle/working\", exist_ok=True)\n\ndf_path = \"/kaggle/working/seq_df_processed.pkl\"\ndf.to_pickle(df_path)\nprint(f\"✅ DataFrame saved to {df_path!r}\")\n\nds_path = \"/kaggle/working/rna_dataset.pkl\"\nwith open(ds_path, \"wb\") as f:\n    pickle.dump(dataset, f)\nprint(f\"✅ Dataset pickled to {ds_path!r}\")\n\ntorch_ds_path = \"/kaggle/working/rna_dataset.pt\"\ntorch.save(dataset, torch_ds_path)\nprint(f\"✅ Dataset torch-saved to {torch_ds_path!r}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-06T16:43:39.781677Z","iopub.execute_input":"2025-05-06T16:43:39.782092Z","iopub.status.idle":"2025-05-06T16:43:40.367787Z","shell.execute_reply.started":"2025-05-06T16:43:39.782062Z","shell.execute_reply":"2025-05-06T16:43:40.366477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}