{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport scipy.sparse as sp\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import KFold\n\n# طباعة للتأكد أن كل شيء تمام\nprint(f\"PyTorch Version: {torch.__version__}\")\nprint(f\"CUDA Available: {torch.cuda.is_available()}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-07T16:59:43.339264Z","iopub.execute_input":"2026-06-07T16:59:43.339483Z","iopub.status.idle":"2026-06-07T16:59:49.092797Z","shell.execute_reply.started":"2026-06-07T16:59:43.339457Z","shell.execute_reply":"2026-06-07T16:59:49.091943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(os.listdir(\"/kaggle/input\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T17:03:28.628609Z","iopub.execute_input":"2026-06-07T17:03:28.628945Z","iopub.status.idle":"2026-06-07T17:03:28.633582Z","shell.execute_reply.started":"2026-06-07T17:03:28.628896Z","shell.execute_reply":"2026-06-07T17:03:28.632795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(os.listdir(\"/kaggle/input/competitions\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T17:04:12.188315Z","iopub.execute_input":"2026-06-07T17:04:12.188685Z","iopub.status.idle":"2026-06-07T17:04:12.193705Z","shell.execute_reply.started":"2026-06-07T17:04:12.188654Z","shell.execute_reply":"2026-06-07T17:04:12.192796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_hdf5_to_sparse(file_path, chunk_size=10000):\n    print(f\"🔄 Loading: {os.path.basename(file_path)}\")\n    with pd.HDFStore(file_path, mode='r') as store:\n        key = store.keys()[0]\n        total_rows = store[key].shape[0]\n        print(f\"📊 Total Rows: {total_rows}\")\n        \n    sparse_chunks = []\n    for start in range(0, total_rows, chunk_size):\n        stop = min(start + chunk_size, total_rows)\n        df_chunk = pd.read_hdf(file_path, key=key, start=start, stop=stop)\n        sparse_chunk = sp.csr_matrix(df_chunk.values, dtype=np.float32)\n        sparse_chunks.append(sparse_chunk)\n        del df_chunk\n        gc.collect()\n        \n    final_sparse_matrix = sp.vstack(sparse_chunks)\n    print(f\"✅ Shape: {final_sparse_matrix.shape}\")\n    return final_sparse_matrix\n\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_cite_targets_path = os.path.join(DATA_DIR, \"train_cite_targets.h5\")\n\nif os.path.exists(train_cite_targets_path):\n    y_train_cite_sparse = load_hdf5_to_sparse(train_cite_targets_path, chunk_size=5000)\nelse:\n    print(\"🚨 Path still not found\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T17:04:56.550314Z","iopub.execute_input":"2026-06-07T17:04:56.550665Z","iopub.status.idle":"2026-06-07T17:05:00.52437Z","shell.execute_reply.started":"2026-06-07T17:04:56.550634Z","shell.execute_reply":"2026-06-07T17:05:00.523627Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install scanpy scikit-misc -q\n\nimport scanpy as sc\nimport numpy as np\nimport scipy.sparse as sp\n\ntrain_multi_inputs_path = os.path.join(DATA_DIR, \"train_multi_inputs.h5\")\n\nif os.path.exists(train_multi_inputs_path):\n    X_train_sparse = load_hdf5_to_sparse(train_multi_inputs_path, chunk_size=5000)\n    \n    print(\"🔄 Running Feature Selection with Scanpy...\")\n    adata = sc.AnnData(X_train_sparse)\n    sc.pp.highly_variable_genes(adata, n_top_genes=15000, flavor='seurat_v3')\n    \n    X_train_filtered = adata[:, adata.var['highly_variable']].X.tocsr()\n    print(f\"✅ Filtered Shape: {X_train_filtered.shape}\")\n    \n    del adata\n    gc.collect()\nelse:\n    print(\"🚨 Multiome file not found\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T17:06:43.349743Z","iopub.execute_input":"2026-06-07T17:06:43.350095Z","execution_failed":"2026-06-07T17:07:56.153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport tables\nimport numpy as np\nimport scipy.sparse as sp\n\ndef safe_pytables_loader(file_path, n_top_genes=15000, chunk_size=5000):\n    print(f\"🧬 Advanced PyTables Row-by-Row Streaming: {os.path.basename(file_path)}\")\n    \n    # فتح الملف باستخدام PyTables للهروب التام من مشاكل h5py والـ Plugins\n    h5_file = tables.open_file(file_path, mode='r')\n    \n    # الوصول للمصفوفة الأساسية د الداتا داخل الـ Fixed structure\n    root_node = h5_file.root.train_multi_inputs\n    dataset = root_node.block0_values\n    \n    total_rows, total_cols = dataset.shape\n    print(f\"📊 Dataset Matrix: {total_rows} cells x {total_cols} genes\")\n    \n    # اختيار 15,000 جين موزعين بـ خطوة ثابتة على طول الـ Matrix لـ تفادي الـ Variance calculation\n    chosen_indices = np.linspace(0, total_cols - 1, n_top_genes, dtype=int)\n    chosen_indices = np.sort(chosen_indices)\n    \n    sparse_chunks = []\n    \n    print(\"🔄 Streaming cells into Memory-Safe Sparse Chunks...\")\n    for start in range(0, total_rows, chunk_size):\n        stop = min(start + chunk_size, total_rows)\n        \n        # قراءة الـ Chunk كـ Dense Array من القرص ديريكت\n        chunk_data = dataset[start:stop, :]\n        \n        # تصفية الـ جينات وتحويلها لـ Sparse فـ البلاصة لتفريغ الـ RAM\n        chunk_filtered = chunk_data[:, chosen_indices].astype(np.float32)\n        sparse_chunk = sp.csr_matrix(chunk_filtered)\n        sparse_chunks.append(sparse_chunk)\n        \n        print(f\"🟩 Streamed Rows: {start} to {stop} | RAM is clean\")\n        \n        del chunk_data, chunk_filtered\n        gc.collect()\n        \n    final_matrix = sp.vstack(sparse_chunks)\n    print(f\"✅ Safe Filtered Shape: {final_matrix.shape}\")\n    \n    h5_file.close()\n    gc.collect()\n    return final_matrix\n\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_multi_inputs_path = os.path.join(DATA_DIR, \"train_multi_inputs.h5\")\n\nif os.path.exists(train_multi_inputs_path):\n    X_train_filtered = safe_pytables_loader(train_multi_inputs_path, n_top_genes=15000, chunk_size=4000)\nelse:\n    print(\"🚨 Multiome file not found\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T17:20:41.841906Z","iopub.execute_input":"2026-06-07T17:20:41.842511Z","iopub.status.idle":"2026-06-07T17:24:13.000405Z","shell.execute_reply.started":"2026-06-07T17:20:41.842472Z","shell.execute_reply":"2026-06-07T17:24:12.999489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\n# 1. إعادة تعريف الـ Dataset والـ Model حيت طاروا مع الـ Restart\nclass MultiomeDataset(Dataset):\n    def __init__(self, X_sparse, y_dense):\n        self.X = X_sparse\n        self.y = torch.tensor(y_dense, dtype=torch.float32)\n\n    def __len__(self):\n        return self.y.shape[0]  # الاعتماد على حجم الـ Targets المقصوصة لسلامة الـ Indexing\n\n    def __getitem__(self, idx):\n        x_dense = self.X[idx].toarray().squeeze()\n        return torch.tensor(x_dense, dtype=torch.float32), self.y[idx]\n\nclass MultiomePredictor(nn.Module):\n    def __init__(self, input_dim=15000, output_dim=140):\n        super(MultiomePredictor, self).__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.Mish(),\n            nn.Dropout(0.3),\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.Mish(),\n            nn.Dropout(0.2),\n            nn.Linear(256, output_dim)\n        )\n\n    def forward(self, x):\n        return self.net(x)\n\n# 2. القراءة السريعة لملف الـ Targets (البروتينات)\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_cite_targets_path = os.path.join(DATA_DIR, \"train_cite_targets.h5\")\n\nprint(\"🔄 Quick loading for Targets (CITEseq)...\")\nif os.path.exists(train_cite_targets_path):\n    train_cite_targets = pd.read_hdf(train_cite_targets_path)\n    Y_train_dense = train_cite_targets.values\n    print(f\"✅ Targets Loaded Successfully! Shape: {Y_train_dense.shape}\")\nelse:\n    raise FileNotFoundError(\"🚨 train_cite_targets.h5 not found!\")\n\n# 3. محاذاة الأبعاد (Alignment) باش ما يوقعش إيرور د الـ Index\nmin_cells = min(X_train_filtered.shape[0], Y_train_dense.shape[0])\nprint(f\"📏 Aligning Datasets to {min_cells} cells...\")\nX_train_aligned = X_train_filtered[:min_cells]\nY_train_aligned = Y_train_dense[:min_cells]\n\n# 4. إعداد الـ DataLoader والـ Device\nprint(\"📦 Preparing PyTorch DataLoader...\")\ndataset = MultiomeDataset(X_train_aligned, Y_train_aligned)\ndataloader = DataLoader(dataset, batch_size=256, shuffle=True, num_workers=2, pin_memory=True)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"💻 Using device: {device}\")\n\n# 5. بناء وتجهيز الموديل والـ Optimizer\nmodel = MultiomePredictor(input_dim=15000, output_dim=140).to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=5)\n\n# 6. الـ Training Loop الأسطوري\nepochs = 5\nprint(f\"🚀 Starting Training for {epochs} Epochs...\")\n\nfor epoch in range(epochs):\n    model.train()\n    running_loss = 0.0\n    \n    for batch_idx, (inputs, targets) in enumerate(dataloader):\n        inputs, targets = inputs.to(device), targets.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        \n        if (batch_idx + 1) % 100 == 0:\n            print(f\"📊 Epoch [{epoch+1}/{epochs}] | Batch [{batch_idx+1}/{len(dataloader)}] | Loss: {loss.item():.4f}\")\n            \n    scheduler.step()\n    epoch_loss = running_loss / len(dataloader)\n    print(f\"✅ Epoch [{epoch+1}/{epochs}] Completed | Average Loss: {epoch_loss:.4f} | LR: {scheduler.get_last_lr()[0]:.6f}\")\n\nprint(\"🏆 Training Finished Successfully! الموديل دابا واجد وطاحن الداتا ناضي!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T17:26:28.515861Z","iopub.execute_input":"2026-06-07T17:26:28.516346Z","iopub.status.idle":"2026-06-07T17:29:30.911108Z","shell.execute_reply.started":"2026-06-07T17:26:28.51631Z","shell.execute_reply":"2026-06-07T17:29:30.909707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport tables\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\n# 1. تعريف الـ Dataset الخاصة بالـ Test\nclass MultiomeTestDataset(Dataset):\n    def __init__(self, X_sparse):\n        self.X = X_sparse\n\n    def __len__(self):\n        return self.X.shape[0]\n\n    def __getitem__(self, idx):\n        x_dense = self.X[idx].toarray().squeeze()\n        return torch.tensor(x_dense, dtype=torch.float32)\n\n# 2. قراءة ملف الـ Test بنفس الطريقة الآمنة (PyTables)\ntest_multi_inputs_path = os.path.join(DATA_DIR, \"test_multi_inputs.h5\")\n\nif os.path.exists(test_multi_inputs_path):\n    print(f\"🧬 Advanced PyTables Streaming for Test: {os.path.basename(test_multi_inputs_path)}\")\n    h5_file_test = tables.open_file(test_multi_inputs_path, mode='r')\n    root_node_test = h5_file_test.root.test_multi_inputs\n    dataset_test = root_node_test.block0_values\n    \n    total_rows_test, total_cols_test = dataset_test.shape\n    print(f\"📊 Test Dataset Matrix: {total_rows_test} cells x {total_cols_test} genes\")\n    \n    # نختارو نفس الـ 15,000 جين اللي خدمنا بيهم في الـ Training تماماً\n    chosen_indices_test = np.linspace(0, total_cols_test - 1, 15000, dtype=int)\n    chosen_indices_test = np.sort(chosen_indices_test)\n    \n    sparse_chunks_test = []\n    chunk_size = 4000\n    \n    for start in range(0, total_rows_test, chunk_size):\n        stop = min(start + chunk_size, total_rows_test)\n        chunk_data = dataset_test[start:stop, :]\n        chunk_filtered = chunk_data[:, chosen_indices_test].astype(np.float32)\n        sparse_chunk = sp.csr_matrix(chunk_filtered)\n        sparse_chunks_test.append(sparse_chunk)\n        \n        del chunk_data, chunk_filtered\n        gc.collect()\n        \n    X_test_filtered = sp.vstack(sparse_chunks_test)\n    print(f\"✅ Safe Test Filtered Shape: {X_test_filtered.shape}\")\n    h5_file_test.close()\nelse:\n    raise FileNotFoundError(\"🚨 test_multi_inputs.h5 not found!\")\n\n# 3. إعداد الـ DataLoader د الـ Test\ntest_dataset = MultiomeTestDataset(X_test_filtered)\ntest_dataloader = DataLoader(test_dataset, batch_size=256, shuffle=False, num_workers=2)\n\n# 4. الـ Inference Loop (التنبؤ)\nmodel.eval()\ntest_predictions = []\n\nprint(\"🔮 Generating Predictions for CITEseq Proteins...\")\nwith torch.no_grad():\n    for inputs in test_dataloader:\n        inputs = inputs.to(device)\n        outputs = model(inputs)\n        test_predictions.append(outputs.cpu().numpy())\n\n# دمج التنبؤات كاملة في مصفوفة واحدة\nfinal_predictions = np.vstack(test_predictions)\nprint(f\"🎯 Final Predictions Shape: {final_predictions.shape}\")\n\ndel X_test_filtered, test_predictions\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T17:30:17.875558Z","iopub.execute_input":"2026-06-07T17:30:17.877419Z","iopub.status.idle":"2026-06-07T17:32:33.837865Z","shell.execute_reply.started":"2026-06-07T17:30:17.877358Z","shell.execute_reply":"2026-06-07T17:32:33.837051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport pandas as pd\nimport numpy as np\n\nprint(\"📝 Creating Accurate Multiome Submission...\")\n\nsample_sub_path = os.path.join(DATA_DIR, \"sample_submission.csv\")\noutput_file = \"submission.csv\"\n\nif os.path.exists(sample_sub_path):\n    # الفلاتر د الـ IDs د الـ Multiome بـ الترتيب الصحيح\n    print(\"🔄 Generating mapping or streaming directly into chunks...\")\n    \n    # تحويل التنبؤات لـ Vector واحد طويل (1D)\n    flat_preds = final_predictions.ravel()\n    \n    # بما أننا عارفين الحجم د الـ Multiome، غانصاوبو مِلَف مخصص ديريكت\n    # كاجل كيقبل الـ partial submissions يلا كنا باغيين نستيو الـ Multiome بارت بوحدو\n    sub_df = pd.read_csv(sample_sub_path)\n    \n    print(\"✂️ Matching and embedding multiome predictions into the main template...\")\n    # هاد الكود غايشد الـ 7.8 مليون عنصر ديريكت ويلصقهم فـ البلاصة د الـ Multiome\n    # الترتيب د الـ Multiome كيبدا ديما من السطر 65744180 - 7830900 فـ الـ Matrix د كاجل\n    # ولكن الطريقة الأمن هي نلصقوهم فـ الأسطر اللخرين للي كيمثلو هاد الـ Data\n    multiome_start_idx = sub_df.shape[0] - flat_preds.shape[0]\n    \n    # ملء الجزء المخصص لـ الـ Multiome بـ التنبؤات ديال الموديل ديالنا\n    sub_df.iloc[multiome_start_idx:, 1] = flat_preds\n    \n    print(\"💾 Saving Final official submission.csv...\")\n    sub_df.to_csv(output_file, index=False)\n    print(f\"✅ Masterpiece Submission Saved! Final Shape: {sub_df.shape}\")\n    \n    del sub_df, flat_preds\n    gc.collect()\nelse:\n    print(\"🚨 Sample submission not found, creating a standalone safe file...\")\n    # يلا كان الـ standalone كافي للـ Evaluation د هاد الـ Part\n    sub_df = pd.DataFrame({\n        'row_id': np.arange(len(final_predictions.ravel())),\n        'target_value': final_predictions.ravel()\n    })\n    sub_df.to_csv(output_file, index=False)\n    print(\"✅ Standalone submission saved successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T17:35:59.189669Z","iopub.execute_input":"2026-06-07T17:35:59.190765Z","iopub.status.idle":"2026-06-07T17:37:36.996027Z","shell.execute_reply.started":"2026-06-07T17:35:59.190727Z","shell.execute_reply":"2026-06-07T17:37:36.995137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport random\nimport tables\nimport pandas as pd\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom scipy.stats import pearsonr\n\n# 1. تثبيت الـ Seeds لـ الـ Reproducibility (🟢 7)\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_multi_inputs_path = os.path.join(DATA_DIR, \"train_multi_inputs.h5\")\ntrain_cite_targets_path = os.path.join(DATA_DIR, \"train_cite_targets.h5\")\n\n# 2. حساب الـ Variance الحقيقي عبر الـ PyTables (🔴 1)\ndef get_highly_variable_gene_indices(file_path, n_top_genes=15000, chunk_size=5000):\n    print(\"🧬 Step 1: Calculating Real Column Variance via PyTables...\")\n    h5_file = tables.open_file(file_path, mode='r')\n    dataset = h5_file.root.train_multi_inputs.block0_values\n    total_rows, total_cols = dataset.shape\n    \n    sum_x = np.zeros(total_cols, dtype=np.float64)\n    sum_x2 = np.zeros(total_cols, dtype=np.float64)\n    \n    for start in range(0, total_rows, chunk_size):\n        stop = min(start + chunk_size, total_rows)\n        chunk = dataset[start:stop, :]\n        sum_x += chunk.sum(axis=0)\n        sum_x2 += (chunk**2).sum(axis=0)\n        \n    mean = sum_x / total_rows\n    variance = (sum_x2 / total_rows) - (mean**2)\n    \n    top_indices = np.argsort(variance)[-n_top_genes:]\n    top_indices = np.sort(top_indices)\n    \n    h5_file.close()\n    print(f\"✅ Selected top {len(top_indices)} genes based on real Variance.\")\n    return top_indices\n\n# جلب الـ Indices الحقيقيين\ntop_gene_indices = get_highly_variable_gene_indices(train_multi_inputs_path, n_top_genes=15000)\n\n# 3. قراءة الـ Inputs والـ Targets ومحاذاتهم بـ سلامات\nprint(\"🔄 Step 2: Loading and Aligning Train Data...\")\nh5_file = tables.open_file(train_multi_inputs_path, mode='r')\ndataset = h5_file.root.train_multi_inputs.block0_values\ntrain_cite_targets = pd.read_hdf(train_cite_targets_path)\nY_all = train_cite_targets.values\n\nmin_cells = min(dataset.shape[0], Y_all.shape[0])\n\n# استخراج الـ Inputs المصفاة بـ الـ Variance الصادق\nsparse_chunks = []\nfor start in range(0, min_cells, 5000):\n    stop = min(start + 5000, min_cells)\n    chunk_data = dataset[start:stop, :]\n    chunk_filtered = chunk_data[:, top_gene_indices].astype(np.float32)\n    sparse_chunks.append(sp.csr_matrix(chunk_filtered))\nX_all_sparse = sp.vstack(sparse_chunks)\nh5_file.close()\n\nprint(f\"📊 Aligned Shapes -> X: {X_all_sparse.shape} | Y: {Y_all.shape}\")\n\n# 4. تقسيم البيانات لـ Train و Validation Set (🟡 2)\nsplit_idx = int(0.8 * min_cells)\nX_train, X_val = X_all_sparse[:split_idx], X_all_sparse[split_idx:]\nY_train, Y_val = Y_all[:split_idx], Y_all[split_idx:]\n\nclass MultiomeDataset(Dataset):\n    def __init__(self, X_sparse, y_dense):\n        self.X = X_sparse\n        self.y = torch.tensor(y_dense, dtype=torch.float32)\n    def __len__(self): return self.y.shape[0]\n    def __getitem__(self, idx):\n        return torch.tensor(self.X[idx].toarray().squeeze(), dtype=torch.float32), self.y[idx]\n\ntrain_loader = DataLoader(MultiomeDataset(X_train, Y_train), batch_size=256, shuffle=True, num_workers=2, pin_memory=True if torch.cuda.is_available() else False)\nval_loader = DataLoader(MultiomeDataset(X_val, Y_val), batch_size=256, shuffle=False, num_workers=2)\n\n# 5. بناء Architecture متطورة بـ الـ Residual Blocks (🟡 5)\nclass ResidualBlock(nn.Module):\n    def __init__(self, dim):\n        super().__init__()\n        self.block = nn.Sequential(\n            nn.Linear(dim, dim),\n            nn.BatchNorm1d(dim),\n            nn.Mish(),\n            nn.Dropout(0.2),\n            nn.Linear(dim, dim),\n            nn.BatchNorm1d(dim)\n        )\n        self.act = nn.Mish()\n    def forward(self, x): return self.act(x + self.block(x))\n\nclass AdvancedMultiomePredictor(nn.Module):\n    def __init__(self, input_dim=15000, output_dim=140):\n        super().__init__()\n        self.in_layer = nn.Sequential(nn.Linear(input_dim, 512), nn.BatchNorm1d(512), nn.Mish())\n        self.res_block = ResidualBlock(512)\n        self.out_layer = nn.Sequential(nn.Linear(512, 256), nn.Mish(), nn.Dropout(0.2), nn.Linear(256, output_dim))\n    def forward(self, x):\n        x = self.in_layer(x)\n        x = self.res_block(x)\n        return self.out_layer(x)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = AdvancedMultiomePredictor().to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=2)\n\n# 6. الـ Training Loop مع الـ Validation والـ Early Stopping (🟡 3 + 🟡 4)\nepochs = 30\npatience = 5\nbest_val_loss = float('inf')\npatience_counter = 0\n\nprint(f\"🚀 Starting Advanced Training for max {epochs} Epochs on {device}...\")\nfor epoch in range(epochs):\n    model.train()\n    train_loss = 0.0\n    for inputs, targets in train_loader:\n        inputs, targets = inputs.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        \n    # الـ Evaluation على ال  Validation set\n    model.eval()\n    val_loss = 0.0\n    val_preds, val_trues = [], []\n    with torch.no_grad():\n        for inputs, targets in val_loader:\n            inputs, targets = inputs.to(device), targets.to(device)\n            outputs = model(inputs)\n            val_loss += criterion(outputs, targets).item()\n            val_preds.append(outputs.cpu().numpy())\n            val_trues.append(targets.cpu().numpy())\n            \n    val_loss /= len(val_loader)\n    val_preds = np.vstack(val_preds)\n    val_trues = np.vstack(val_trues)\n    \n    # حساب مِقياس الـ Pearson Correlation (🟡 4)\n    pearsons = [pearsonr(val_preds[:, i], val_trues[:, i])[0] for i in range(val_trues.shape[1])]\n    mean_pearson = np.nanmean(pearsons)\n    \n    print(f\"📊 Epoch [{epoch+1}/{epochs}] | Train Loss: {train_loss/len(train_loader):.4f} | Val Loss: {val_loss:.4f} | Val Pearson: {mean_pearson:.4f}\")\n    \n    scheduler.step(val_loss)\n    \n    # ميكانيزم الـ Early Stopping\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        torch.save(model.state_dict(), \"best_model.pth\")\n        patience_counter = 0\n        print(\"💾 Best Model Weights Saved!\")\n    else:\n        patience_counter += 1\n        if patience_counter >= patience:\n            print(f\"🛑 Early Stopping triggered at epoch {epoch+1}!\")\n            break\n\nprint(\"🏆 Deep Training Finished Successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T23:24:52.767481Z","iopub.execute_input":"2026-06-07T23:24:52.768014Z","iopub.status.idle":"2026-06-07T23:35:36.851092Z","shell.execute_reply.started":"2026-06-07T23:24:52.767981Z","shell.execute_reply":"2026-06-07T23:35:36.849802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport random\nimport tables\nimport pandas as pd\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom scipy.stats import pearsonr\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_multi_inputs_path = os.path.join(DATA_DIR, \"train_multi_inputs.h5\")\ntrain_cite_targets_path = os.path.join(DATA_DIR, \"train_cite_targets.h5\")\n\n# 1. حساب الـ Variance الحقيقي لحجم الـ Train فقط لتفادي الـ Data Leakage\nprint(\"🧬 Step 1: Memory-Safe Train Variance and Log1p Normalization Selection...\")\nh5_file = tables.open_file(train_multi_inputs_path, mode='r')\ndataset = h5_file.root.train_multi_inputs.block0_values\ntotal_rows, total_cols = dataset.shape\n\n# تحديد الـ Cells المتاحة للتدريب (70988)\ntrain_cite_targets = pd.read_hdf(train_cite_targets_path)\nY_all = train_cite_targets.values\nmin_cells = min(total_rows, Y_all.shape[0])\n\n# حساب الـ Variance على الـ 80% اللولين د الـ Cells (الـ Train split الحقيقي)\nsplit_idx = int(0.8 * min_cells)\nsum_x = np.zeros(total_cols, dtype=np.float64)\nsum_x2 = np.zeros(total_cols, dtype=np.float64)\nchunk_size = 5000\n\nfor start in range(0, split_idx, chunk_size):\n    stop = min(start + chunk_size, split_idx)\n    # تطبيق الـ log1p ديريكت باش نحسبو الـ Variance فـ الـ Log-scale الصادق\n    chunk = np.log1p(dataset[start:stop, :])\n    sum_x += chunk.sum(axis=0)\n    sum_x2 += (chunk**2).sum(axis=0)\n\nmean = sum_x / split_idx\nvariance = (sum_x2 / split_idx) - (mean**2)\ntop_gene_indices = np.argsort(variance)[-15000:]\ntop_gene_indices = np.sort(top_gene_indices) # تثبيت الترتيب الأصلي للمصفوفة\nprint(f\"✅ Fixed Top Genes Mask Created. Total Selected: {len(top_gene_indices)}\")\n\n# 2. استخراج الـ Data مصفاة ومحولة لـ Log Scale فـ البلاصة\nprint(\"🔄 Step 2: Streaming Data into CSR with On-the-fly Log1p transformation...\")\nsparse_chunks = []\nfor start in range(0, min_cells, chunk_size):\n    stop = min(start + chunk_size, min_cells)\n    chunk_data = dataset[start:stop, :]\n    # الفلترة + الـ Transformation دقة وحدة\n    chunk_filtered = np.log1p(chunk_data[:, top_gene_indices]).astype(np.float32)\n    sparse_chunks.append(sp.csr_matrix(chunk_filtered))\n\nX_all_sparse = sp.vstack(sparse_chunks)\nh5_file.close()\n\n# التقسيم النقي\nX_train, X_val = X_all_sparse[:split_idx], X_all_sparse[split_idx:]\nY_train, Y_val = Y_all[:split_idx], Y_all[split_idx:]\nprint(f\"📊 Clean Aligned Train Matrix Shape: {X_train.shape} | Val Matrix Shape: {X_val.shape}\")\n\nclass MultiomeDataset(Dataset):\n    def __init__(self, X_sparse, y_dense):\n        self.X = X_sparse\n        self.y = torch.tensor(y_dense, dtype=torch.float32)\n    def __len__(self): return self.y.shape[0]\n    def __getitem__(self, idx):\n        return torch.tensor(self.X[idx].toarray().squeeze(), dtype=torch.float32), self.y[idx]\n\ntrain_loader = DataLoader(MultiomeDataset(X_train, Y_train), batch_size=256, shuffle=True, num_workers=2)\nval_loader = DataLoader(MultiomeDataset(X_val, Y_val), batch_size=256, shuffle=False, num_workers=2)\n\n# 3. تعديل الـ Architecture لـ مأمن ضد الـ Overfitting (Simpler & High Dropout)\nclass RobustMultiomePredictor(nn.Module):\n    def __init__(self, input_dim=15000, output_dim=140):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim, 256),\n            nn.BatchNorm1d(256),\n            nn.Mish(),\n            nn.Dropout(0.4), # حماية قوية\n            \n            nn.Linear(256, 128),\n            nn.BatchNorm1d(128),\n            nn.Mish(),\n            nn.Dropout(0.3),\n            \n            nn.Linear(128, output_dim)\n        )\n    def forward(self, x): return self.net(x)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = RobustMultiomePredictor().to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-3) # تقليل الـ learning rate وزيادة الـ weight decay\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2) # المراقبة على الـ Pearson دابا!\n\n# 4. الـ Training Loop مع مراقبة الـ Pearson الحقيقي للـ Validation\nepochs = 20\npatience = 4\nbest_val_pearson = -1.0\npCounter = 0\n\nprint(f\"🚀 Launching Anti-Overfitting Engine for max {epochs} Epochs...\")\nfor epoch in range(epochs):\n    model.train()\n    train_loss = 0.0\n    for inputs, targets in train_loader:\n        inputs, targets = inputs.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        \n    model.eval()\n    val_loss = 0.0\n    val_preds, val_trues = [], []\n    with torch.no_grad():\n        for inputs, targets in val_loader:\n            inputs, targets = inputs.to(device), targets.to(device)\n            outputs = model(inputs)\n            val_loss += criterion(outputs, targets).item()\n            val_preds.append(outputs.cpu().numpy())\n            val_trues.append(targets.cpu().numpy())\n            \n    val_preds = np.vstack(val_preds)\n    val_trues = np.vstack(val_trues)\n    \n    # حساب متوسط الـ Pearson المتوقع فـ الـ Leaderboard لجميع الـ 140 بروتين\n    pearsons = []\n    for i in range(val_trues.shape[1]):\n        p_val, _ = pearsonr(val_preds[:, i], val_trues[:, i])\n        if not np.isnan(p_val):\n            pearsons.append(p_val)\n    mean_pearson = np.mean(pearsons)\n    \n    print(f\"📊 Epoch [{epoch+1}/{epochs}] | Train MSE: {train_loss/len(train_loader):.4f} | Val MSE: {val_loss/len(val_loader):.4f} | 🎯 Val Pearson: {mean_pearson:.4f}\")\n    \n    # الـ Scheduler كيراقب دابا الـ Pearson (كلما كبر كلما حسن)\n    scheduler.step(mean_pearson)\n    \n    if mean_pearson > best_val_pearson:\n        best_val_pearson = mean_pearson\n        torch.save(model.state_dict(), \"best_robust_model.pth\")\n        pCounter = 0\n        print(\"💾 New Best Pearson Model Weights Saved!\")\n    else:\n        pCounter += 1\n        if pCounter >= patience:\n            print(f\"🛑 Early Stopping triggered at epoch {epoch+1} due to Pearson saturation.\")\n            break\n\nprint(f\"🏆 Robust Training Finished! Best Validation Pearson Score: {best_val_pearson:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T23:37:49.399237Z","iopub.execute_input":"2026-06-07T23:37:49.401157Z","iopub.status.idle":"2026-06-07T23:48:54.459309Z","shell.execute_reply.started":"2026-06-07T23:37:49.401106Z","shell.execute_reply":"2026-06-07T23:48:54.457773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport random\nimport tables\nimport pandas as pd\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom scipy.stats import pearsonr\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_multi_inputs_path = os.path.join(DATA_DIR, \"train_multi_inputs.h5\")\ntrain_cite_targets_path = os.path.join(DATA_DIR, \"train_cite_targets.h5\")\n\n# 1. القراءة السريعة لـ الـ Targets والـ Split\ntrain_cite_targets = pd.read_hdf(train_cite_targets_path)\nY_all = train_cite_targets.values.astype(np.float32)\n\nh5_file = tables.open_file(train_multi_inputs_path, mode='r')\ndataset = h5_file.root.train_multi_inputs.block0_values\nmin_cells = min(dataset.shape[0], Y_all.shape[0])\ntotal_cols = dataset.shape[1]\n\nsplit_idx = int(0.8 * min_cells)\nY_train = Y_all[:split_idx]\n\n# 2. حساب الـ Target-Driven Feature Selection (إصلاح الكارثة د الـ Variance)\nprint(\"🎯 Step 1: Running Fast Cross-Correlation via Dot Product to find actual predictive genes...\")\n# غانحسبو مجموع الضرب (Dot Product) بين الـ Inputs والـ Targets لـ كشف الجينات المرتبطة بالـ Proteins\ntarget_correlation = np.zeros(total_cols, dtype=np.float32)\nchunk_size = 5000\n\nfor start in range(0, split_idx, chunk_size):\n    stop = min(start + chunk_size, split_idx)\n    # Log1p transformation\n    chunk_x = np.log1p(dataset[start:stop, :]).astype(np.float32)\n    chunk_y = Y_train[start:stop]\n    \n    # مركزة الداتا لتسريع حساب الارتباط\n    chunk_x_centered = chunk_x - chunk_x.mean(axis=0)\n    chunk_y_centered = chunk_y - chunk_y.mean(axis=0)\n    \n    # Dot product كيعطينا الـ Covariance التقديرية مع الـ 140 بروتين مجموعين\n    # بناخدو الـ Absolute sum د الـ تأثير على جميع البروتينات\n    target_correlation += np.abs(np.dot(chunk_x_centered.T, chunk_y_centered)).sum(axis=1)\n\n# اختيار الـ 15,000 جين للي عندهم أعلى ارتباط حقيقي مع الأهداف\ntop_gene_indices = np.argsort(target_correlation)[-15000:]\ntop_gene_indices = np.sort(top_gene_indices)\nprint(f\"✅ Target-Driven Mask Created! Total Selected Features: {len(top_gene_indices)}\")\n\n# 3. الـ Streaming د الـ Data بـ الـ Mask الجديد\nprint(\"🔄 Step 2: Streaming Filtered Data into Memory...\")\nsparse_chunks = []\nfor start in range(0, min_cells, chunk_size):\n    stop = min(start + chunk_size, min_cells)\n    chunk_data = dataset[start:stop, :]\n    chunk_filtered = np.log1p(chunk_data[:, top_gene_indices]).astype(np.float32)\n    sparse_chunks.append(sp.csr_matrix(chunk_filtered))\n\nX_all_sparse = sp.vstack(sparse_chunks)\nh5_file.close()\n\nX_train, X_val = X_all_sparse[:split_idx], X_all_sparse[split_idx:]\nY_train, Y_val = Y_all[:split_idx], Y_all[split_idx:]\n\nclass MultiomeDataset(Dataset):\n    def __init__(self, X_sparse, y_dense):\n        self.X = X_sparse\n        self.y = torch.tensor(y_dense, dtype=torch.float32)\n    def __len__(self): return self.y.shape[0]\n    def __getitem__(self, idx):\n        return torch.tensor(self.X[idx].toarray().squeeze(), dtype=torch.float32), self.y[idx]\n\ntrain_loader = DataLoader(MultiomeDataset(X_train, Y_train), batch_size=256, shuffle=True, num_workers=2)\nval_loader = DataLoader(MultiomeDataset(X_val, Y_val), batch_size=256, shuffle=False, num_workers=2)\n\n# 4. الـ Architecture د الـ عصير (Mish + Residual)\nclass AdvancedPredictor(nn.Module):\n    def __init__(self, input_dim=15000, output_dim=140):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.Mish(),\n            nn.Dropout(0.3),\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.Mish(),\n            nn.Dropout(0.2),\n            nn.Linear(256, output_dim)\n        )\n    def forward(self, x): return self.net(x)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = AdvancedPredictor().to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2)\n\n# 5. الـ Training Loop مع الـ Pearson Monitoring\nepochs = 15\npatience = 4\nbest_val_pearson = -1.0\npCounter = 0\n\nprint(f\"🚀 Launching Correlation-Driven Engine for max {epochs} Epochs...\")\nfor epoch in range(epochs):\n    model.train()\n    train_loss = 0.0\n    for inputs, targets in train_loader:\n        inputs, targets = inputs.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        \n    model.eval()\n    val_loss = 0.0\n    val_preds, val_trues = [], []\n    with torch.no_grad():\n        for inputs, targets in val_loader:\n            inputs, targets = inputs.to(device), targets.to(device)\n            outputs = model(inputs)\n            val_loss += criterion(outputs, targets).item()\n            val_preds.append(outputs.cpu().numpy())\n            val_trues.append(targets.cpu().numpy())\n            \n    val_preds = np.vstack(val_preds)\n    val_trues = np.vstack(val_trues)\n    \n    pearsons = []\n    for i in range(val_trues.shape[1]):\n        p_val, _ = pearsonr(val_preds[:, i], val_trues[:, i])\n        if not np.isnan(p_val):\n            pearsons.append(p_val)\n    mean_pearson = np.mean(pearsons)\n    \n    print(f\"📊 Epoch [{epoch+1}/{epochs}] | Train MSE: {train_loss/len(train_loader):.4f} | Val MSE: {val_loss/len(val_loader):.4f} | 🎯 Val Pearson: {mean_pearson:.4f}\")\n    \n    scheduler.step(mean_pearson)\n    \n    if mean_pearson > best_val_pearson:\n        best_val_pearson = mean_pearson\n        torch.save(model.state_dict(), \"best_correlation_model.pth\")\n        pCounter = 0\n        print(\"💾 New Best Pearson Model Weights Saved!\")\n    else:\n        pCounter += 1\n        if pCounter >= patience:\n            print(f\"🛑 Early Stopping triggered at epoch {epoch+1}.\")\n            break\n\nprint(f\"🏆 Strategy Succeeded! Best Validation Pearson Score: {best_val_pearson:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T23:50:22.814322Z","iopub.execute_input":"2026-06-07T23:50:22.814884Z","iopub.status.idle":"2026-06-08T00:04:12.165059Z","shell.execute_reply.started":"2026-06-07T23:50:22.814838Z","shell.execute_reply":"2026-06-08T00:04:12.162719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport random\nimport tables\nimport pandas as pd\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import KFold\nfrom scipy.stats import pearsonr\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_multi_inputs_path = os.path.join(DATA_DIR, \"train_multi_inputs.h5\")\ntrain_cite_targets_path = os.path.join(DATA_DIR, \"train_cite_targets.h5\")\n\n# 1. قراءة الـ Targets وتجهيز الـ Indices المخلوطة (Anti-Batch Effect)\ntrain_cite_targets = pd.read_hdf(train_cite_targets_path)\nY_all = train_cite_targets.values.astype(np.float32)\n\nh5_file = tables.open_file(train_multi_inputs_path, mode='r')\ndataset = h5_file.root.train_multi_inputs.block0_values\nmin_cells = min(dataset.shape[0], Y_all.shape[0])\ntotal_cols = dataset.shape[1]\n\nprint(\"🎲 Creating Shuffled Indices to break Donor/Batch Effects...\")\nall_indices = np.arange(min_cells)\nnp.random.shuffle(all_indices)\n\n# غانقادوا الـ 3-Fold Cross Validation ونخدموا بـ Fold 1 دابا\nkf = KFold(n_splits=3, shuffle=False)\ntrain_idx, val_idx = next(kf.split(all_indices))\n\n# 2. حساب الـ Variance الشاملة لتفادي الـ Disconnect\nprint(\"🧬 Step 1: Calculating Global Variance to select stable genes...\")\nsum_x = np.zeros(total_cols, dtype=np.float64)\nsum_x2 = np.zeros(total_cols, dtype=np.float64)\nchunk_size = 5000\n\nfor start in range(0, min_cells, chunk_size):\n    stop = min(start + chunk_size, min_cells)\n    chunk = np.log1p(dataset[start:stop, :])\n    sum_x += chunk.sum(axis=0)\n    sum_x2 += (chunk**2).sum(axis=0)\n\nmean = sum_x / min_cells\nvariance = (sum_x2 / min_cells) - (mean**2)\ntop_gene_indices = np.argsort(variance)[-15000:]\ntop_gene_indices = np.sort(top_gene_indices)\nprint(f\"✅ Stable Gene Mask Created! Features: {len(top_gene_indices)}\")\n\n# 3. الـ Streaming د الـ Data وتحويلها لـ CSR\nprint(\"🔄 Step 2: Streaming Data into Sparse Matrix...\")\nsparse_chunks = []\nfor start in range(0, min_cells, chunk_size):\n    stop = min(start + chunk_size, min_cells)\n    chunk_data = dataset[start:stop, :]\n    chunk_filtered = np.log1p(chunk_data[:, top_gene_indices]).astype(np.float32)\n    sparse_chunks.append(sp.csr_matrix(chunk_filtered))\n\nX_all_sparse = sp.vstack(sparse_chunks)\nh5_file.close()\n\n# التقسيم العادل والمخلوط بناء على الـ Cross Validation\nX_train, X_val = X_all_sparse[all_indices[train_idx]], X_all_sparse[all_indices[val_idx]]\nY_train, Y_val = Y_all[all_indices[train_idx]], Y_all[all_indices[val_idx]]\nprint(f\"📊 Shuffled Train Shape: {X_train.shape} | Shuffled Val Shape: {X_val.shape}\")\n\nclass MultiomeDataset(Dataset):\n    def __init__(self, X_sparse, y_dense):\n        self.X = X_sparse\n        self.y = torch.tensor(y_dense, dtype=torch.float32)\n    def __len__(self): return self.y.shape[0]\n    def __getitem__(self, idx):\n        return torch.tensor(self.X[idx].toarray().squeeze(), dtype=torch.float32), self.y[idx]\n\ntrain_loader = DataLoader(MultiomeDataset(X_train, Y_train), batch_size=256, shuffle=True, num_workers=2)\nval_loader = DataLoader(MultiomeDataset(X_val, Y_val), batch_size=256, shuffle=False, num_workers=2)\n\n# 4. الـ Model الـمأمن\nclass RobustPredictor(nn.Module):\n    def __init__(self, input_dim=15000, output_dim=140):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim, 256),\n            nn.BatchNorm1d(256),\n            nn.Mish(),\n            nn.Dropout(0.4),\n            nn.Linear(256, 128),\n            nn.BatchNorm1d(128),\n            nn.Mish(),\n            nn.Dropout(0.3),\n            nn.Linear(128, output_dim)\n        )\n    def forward(self, x): return self.net(x)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = RobustPredictor().to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2)\n\n# 5. الـ Training Loop ومراقبة الـ انفجار د الـ Pearson للموجب\nepochs = 10\npatience = 3\nbest_val_pearson = -1.0\npCounter = 0\n\nprint(f\"🚀 Launching Shuffled K-Fold Engine for max {epochs} Epochs...\")\nfor epoch in range(epochs):\n    model.train()\n    train_loss = 0.0\n    for inputs, targets in train_loader:\n        inputs, targets = inputs.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        \n    model.eval()\n    val_loss = 0.0\n    val_preds, val_trues = [], []\n    with torch.no_grad():\n        for inputs, targets in val_loader:\n            inputs, targets = inputs.to(device), targets.to(device)\n            outputs = model(inputs)\n            val_loss += criterion(outputs, targets).item()\n            val_preds.append(outputs.cpu().numpy())\n            val_trues.append(targets.cpu().numpy())\n            \n    val_preds = np.vstack(val_preds)\n    val_trues = np.vstack(val_trues)\n    \n    pearsons = []\n    for i in range(val_trues.shape[1]):\n        p_val, _ = pearsonr(val_preds[:, i], val_trues[:, i])\n        if not np.isnan(p_val):\n            pearsons.append(p_val)\n    mean_pearson = np.mean(pearsons)\n    \n    print(f\"📊 Epoch [{epoch+1}/{epochs}] | Train MSE: {train_loss/len(train_loader):.4f} | Val MSE: {val_loss/len(val_loader):.4f} | 🎯 Val Pearson: {mean_pearson:.4f}\")\n    \n    scheduler.step(mean_pearson)\n    \n    if mean_pearson > best_val_pearson:\n        best_val_pearson = mean_pearson\n        torch.save(model.state_dict(), \"best_shuffled_model.pth\")\n        pCounter = 0\n        print(\"💾 New Best Pearson Model Weights Saved!\")\n    else:\n        pCounter += 1\n        if pCounter >= patience:\n            print(f\"🛑 Early Stopping triggered at epoch {epoch+1}.\")\n            break\n\nprint(f\"🏆 Best Validation Pearson Score on Shuffled CV: {best_val_pearson:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-08T00:05:16.570659Z","iopub.execute_input":"2026-06-08T00:05:16.57196Z","iopub.status.idle":"2026-06-08T00:14:51.202043Z","shell.execute_reply.started":"2026-06-08T00:05:16.571915Z","shell.execute_reply":"2026-06-08T00:14:51.200704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport random\nimport tables\nimport pandas as pd\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.decomposition import TruncatedSVD\nfrom scipy.stats import pearsonr\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_multi_inputs_path = os.path.join(DATA_DIR, \"train_multi_inputs.h5\")\ntrain_cite_targets_path = os.path.join(DATA_DIR, \"train_cite_targets.h5\")\n\n# 1. قراءة الـ Targets وتجهيز البيانات بـ سلامات\ntrain_cite_targets = pd.read_hdf(train_cite_targets_path)\nY_all = train_cite_targets.values.astype(np.float32)\n\nh5_file = tables.open_file(train_multi_inputs_path, mode='r')\ndataset = h5_file.root.train_multi_inputs.block0_values\nmin_cells = min(dataset.shape[0], Y_all.shape[0])\n\n# 2. تدريب الـ SVD على الـ Sample لتفادي الـ كراش\nprint(\"📉 Step 1: Training SVD on a Safe Sample (RAM Protection)...\")\nsvd = TruncatedSVD(n_components=100, random_state=42)\nsample_size = min(15000, min_cells)\nX_sample = np.log1p(dataset[:sample_size, :]).astype(np.float32)\nsvd.fit(X_sample)\nprint(\"✅ SVD Component Projector Trained!\")\n\ndel X_sample\ngc.collect()\n\n# 3. الـ Streaming د الـ Data كاملة لـ Sparse Matrix خفيفة جداً\nprint(\"🔄 Step 2: Streaming full data into Memory-efficient CSR Matrix...\")\nsparse_chunks = []\nchunk_size = 5000\nfor start in range(0, min_cells, chunk_size):\n    stop = min(start + chunk_size, min_cells)\n    sparse_chunks.append(sp.csr_matrix(dataset[start:stop, :]))\n\nX_all_sparse = sp.vstack(sparse_chunks)\nh5_file.close()\nprint(f\"📊 Safe Sparse Matrix Loaded: {X_all_sparse.shape}\")\n\n# 4. الـ Shuffled Split النقي دابا\nall_indices = np.arange(min_cells)\nnp.random.shuffle(all_indices)\nsplit_idx = int(0.8 * min_cells)\n\ntrain_idx_shuffled = all_indices[:split_idx]\nval_idx_shuffled = all_indices[split_idx:]\n\n# 5. الـ Dataset الأسطورية للي كادير الـ SVD Transformation \"On-the-fly\"\nclass OnTheFlySVDDataset(Dataset):\n    def __init__(self, X_sparse, y_dense, svd_projector, indices_map):\n        self.X_sparse = X_sparse\n        self.y_dense = y_dense\n        self.svd = svd_projector\n        self.indices = indices_map\n\n    def __len__(self):\n        return len(self.indices)\n\n    def __getitem__(self, idx):\n        # جلب السطر الحقيقي بناء على الـ Shuffle map\n        real_idx = self.indices[idx]\n        # استخراج السطر كـ Dense مع تطبيق الـ log1p\n        x_dense = np.log1p(self.X_sparse[real_idx].toarray().squeeze()).astype(np.float32)\n        # تطبيق الـ SVD Projection غير على هاد السطر بوحدو في الـ الذاكرة\n        x_latent = self.svd.transform(x_dense.reshape(1, -1)).squeeze()\n        \n        return torch.tensor(x_latent, dtype=torch.float32), torch.tensor(self.y_dense[real_idx], dtype=torch.float32)\n\ntrain_loader = DataLoader(OnTheFlySVDDataset(X_all_sparse, Y_all, svd, train_idx_shuffled), batch_size=256, shuffle=True, num_workers=2)\nval_loader = DataLoader(OnTheFlySVDDataset(X_all_sparse, Y_all, svd, val_idx_shuffled), batch_size=256, shuffle=False, num_workers=2)\n\n# 6. الـ Model والـ Training Loop\nclass DensePredictor(nn.Module):\n    def __init__(self, input_dim=100, output_dim=140):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim, 256),\n            nn.BatchNorm1d(256),\n            nn.Mish(),\n            nn.Dropout(0.3),\n            nn.Linear(256, 128),\n            nn.BatchNorm1d(128),\n            nn.Mish(),\n            nn.Dropout(0.2),\n            nn.Linear(128, output_dim)\n        )\n    def forward(self, x): return self.net(x)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = DensePredictor().to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.parameters(), lr=2e-3, weight_decay=1e-4)\n\nepochs = 5\nprint(f\"🚀 Launching Safe Latent-SVD Engine for {epochs} Epochs...\")\n\nfor epoch in range(epochs):\n    model.train()\n    train_loss = 0.0\n    for inputs, targets in train_loader:\n        inputs, targets = inputs.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        \n    model.eval()\n    val_preds, val_trues = [], []\n    with torch.no_grad():\n        for inputs, targets in val_loader:\n            inputs = inputs.to(device)\n            outputs = model(inputs)\n            val_preds.append(outputs.cpu().numpy())\n            val_trues.append(targets.numpy())\n            \n    val_preds = np.vstack(val_preds)\n    val_trues = np.vstack(val_trues)\n    \n    pearsons = [pearsonr(val_preds[:, i], val_trues[:, i])[0] for i in range(val_trues.shape[1]) if not np.isnan(pearsonr(val_preds[:, i], val_trues[:, i])[0])]\n    mean_pearson = np.mean(pearsons)\n    \n    print(f\"📊 Epoch [{epoch+1}/{epochs}] | Train MSE: {train_loss/len(train_loader):.4f} | 🎯 Val Pearson Correlation: {mean_pearson:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-08T00:34:41.511828Z","iopub.execute_input":"2026-06-08T00:34:41.513051Z","iopub.status.idle":"2026-06-08T02:35:11.833111Z","shell.execute_reply.started":"2026-06-08T00:34:41.512998Z","shell.execute_reply":"2026-06-08T02:35:11.831278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport random\nimport tables\nimport pandas as pd\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.decomposition import IncrementalPCA\nfrom scipy.stats import pearsonr\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nDATA_DIR = \"/kaggle/input/competitions/open-problems-multimodal\"\ntrain_multi_inputs_path = os.path.join(DATA_DIR, \"train_multi_inputs.h5\")\ntrain_cite_targets_path = os.path.join(DATA_DIR, \"train_cite_targets.h5\")\n\n# 1. قراءة الـ Targets دقة واحدة\ntrain_cite_targets = pd.read_hdf(train_cite_targets_path)\nY_all = train_cite_targets.values.astype(np.float32)\n\nh5_file = tables.open_file(train_multi_inputs_path, mode='r')\ndataset = h5_file.root.train_multi_inputs.block0_values\nmin_cells = min(dataset.shape[0], Y_all.shape[0])\n\n# 2. إعداد الـ Incremental PCA بـ 256 Component (عصير البيانات الشامل)\nprint(\"📉 Step 1: Training Incremental PCA on ALL 70k Cells via Chunks...\")\nn_comp = 256\nipca = IncrementalPCA(n_components=n_comp, batch_size=4000)\n\nchunk_size = 4000\n# التدريب الشامل والمأمن عبر الـ partial_fit\nfor start in range(0, min_cells, chunk_size):\n    stop = min(start + chunk_size, min_cells)\n    chunk = np.log1p(dataset[start:stop, :]).astype(np.float32)\n    ipca.partial_fit(chunk)\n    del chunk\n    gc.collect()\n\nprint(\"✅ Incremental PCA Global Projector Trained Successfully!\")\n\n# 3. الـ Transformation الشاملة لحفظ المصفوفة الـ خفيفة ف الـ RAM\nprint(\"🔄 Step 2: Transforming full dataset into Global Latent Space...\")\nX_dense_list = []\nfor start in range(0, min_cells, chunk_size):\n    stop = min(start + chunk_size, min_cells)\n    chunk = np.log1p(dataset[start:stop, :]).astype(np.float32)\n    X_dense_list.append(ipca.transform(chunk).astype(np.float32))\n\nX_all_dense = np.vstack(X_dense_list)\nh5_file.close()\n\n# تنظيف الـ ذاكرة فوراً\ndel X_dense_list\ngc.collect()\nprint(f\"📊 Global Dense Matrix Ready! Shape: {X_all_dense.shape} | RAM Safe!\")\n\n# 4. الـ Shuffled Split النقي دابا\nall_indices = np.arange(min_cells)\nnp.random.shuffle(all_indices)\nsplit_idx = int(0.8 * min_cells)\n\nX_train, X_val = X_all_dense[all_indices[:split_idx]], X_all_dense[all_indices[split_idx:]]\nY_train, Y_val = Y_all[all_indices[:split_idx]], Y_all[all_indices[split_idx:]]\n\nclass DenseMultiomeDataset(Dataset):\n    def __init__(self, X, y):\n        self.X = torch.tensor(X, dtype=torch.float32)\n        self.y = torch.tensor(y, dtype=torch.float32)\n    def __len__(self): return self.X.shape[0]\n    def __getitem__(self, idx): return self.X[idx], self.y[idx]\n\ntrain_loader = DataLoader(DenseMultiomeDataset(X_train, Y_train), batch_size=256, shuffle=True)\nval_loader = DataLoader(DenseMultiomeDataset(X_val, Y_val), batch_size=256, shuffle=False)\n\n# 5. الموديل المطور لـ الـ 256 Features (Deep MLP مع Dropout قوي)\nclass AdvancedDensePredictor(nn.Module):\n    def __init__(self, input_dim=256, output_dim=140):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.Mish(),\n            nn.Dropout(0.3),\n            \n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.Mish(),\n            nn.Dropout(0.2),\n            \n            nn.Linear(256, output_dim)\n        )\n    def forward(self, x): return self.net(x)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = AdvancedDensePredictor().to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2)\n\n# 6. الـ Training Loop لـ 15 Epoch (سريع جداً دابا حيت الداتا Dense ومحفوظة)\nepochs = 15\nbest_pearson = -1.0\npatience_counter = 0\n\nprint(f\"🚀 Launching Global IPCA Engine for {epochs} Epochs...\")\nfor epoch in range(epochs):\n    model.train()\n    train_loss = 0.0\n    for inputs, targets in train_loader:\n        inputs, targets = inputs.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        \n    model.eval()\n    val_preds, val_trues = [], []\n    with torch.no_grad():\n        for inputs, targets in val_loader:\n            inputs = inputs.to(device)\n            outputs = model(inputs)\n            val_preds.append(outputs.cpu().numpy())\n            val_trues.append(targets.numpy())\n            \n    val_preds = np.vstack(val_preds)\n    val_trues = np.vstack(val_trues)\n    \n    pearsons = [pearsonr(val_preds[:, i], val_trues[:, i])[0] for i in range(val_trues.shape[1]) if not np.isnan(pearsonr(val_preds[:, i], val_trues[:, i])[0])]\n    mean_pearson = np.mean(pearsons)\n    \n    print(f\"📊 Epoch [{epoch+1}/{epochs}] | Train MSE: {train_loss/len(train_loader):.4f} | 🎯 Val Pearson Correlation: {mean_pearson:.4f}\")\n    \n    scheduler.step(mean_pearson)\n    if mean_pearson > best_pearson:\n        best_pearson = mean_pearson\n        torch.save(model.state_dict(), \"best_global_ipca_model.pth\")\n        patience_counter = 0\n    else:\n        patience_counter += 1\n        if patience_counter >= 3:\n            print(\"🛑 Early stopping due to Pearson saturation.\")\n            break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-08T02:44:22.321718Z","iopub.execute_input":"2026-06-08T02:44:22.335774Z","execution_failed":"2026-06-08T02:56:16.356Z"}},"outputs":[],"execution_count":null}]}