{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":87793,"databundleVersionId":12024591,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:26.717237Z","iopub.execute_input":"2025-05-10T13:53:26.717572Z","iopub.status.idle":"2025-05-10T13:53:27.015626Z","shell.execute_reply.started":"2025-05-10T13:53:26.717543Z","shell.execute_reply":"2025-05-10T13:53:27.014987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import statistics\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader,  Subset, random_split\nimport torch.optim as optim\nfrom tqdm import tqdm\nimport torch\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\nfrom sklearn.compose import ColumnTransformer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:27.016560Z","iopub.execute_input":"2025-05-10T13:53:27.016930Z","iopub.status.idle":"2025-05-10T13:53:32.164227Z","shell.execute_reply.started":"2025-05-10T13:53:27.016898Z","shell.execute_reply":"2025-05-10T13:53:32.163538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels = pd.read_csv(\"/kaggle/input/stanford-rna-3d-folding/train_labels.csv\")\ntrain_sequence = pd.read_csv(\"/kaggle/input/stanford-rna-3d-folding/train_sequences.csv\")\nval_labels = pd.read_csv(\"/kaggle/input/stanford-rna-3d-folding/validation_labels.csv\")\nval_sequence = pd.read_csv(\"/kaggle/input/stanford-rna-3d-folding/validation_sequences.csv\")\ntest_sequence = pd.read_csv(\"/kaggle/input/stanford-rna-3d-folding/test_sequences.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:32.165480Z","iopub.execute_input":"2025-05-10T13:53:32.165967Z","iopub.status.idle":"2025-05-10T13:53:32.529097Z","shell.execute_reply.started":"2025-05-10T13:53:32.165942Z","shell.execute_reply":"2025-05-10T13:53:32.527600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels[\"resid\"].idxmax()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:32.531530Z","iopub.execute_input":"2025-05-10T13:53:32.532013Z","iopub.status.idle":"2025-05-10T13:53:32.543414Z","shell.execute_reply.started":"2025-05-10T13:53:32.531947Z","shell.execute_reply":"2025-05-10T13:53:32.542588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels[\"resid\"].max()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:32.544362Z","iopub.execute_input":"2025-05-10T13:53:32.544631Z","iopub.status.idle":"2025-05-10T13:53:32.571336Z","shell.execute_reply.started":"2025-05-10T13:53:32.544598Z","shell.execute_reply":"2025-05-10T13:53:32.570719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"maxlength=4300","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:32.572340Z","iopub.execute_input":"2025-05-10T13:53:32.572640Z","iopub.status.idle":"2025-05-10T13:53:32.586119Z","shell.execute_reply.started":"2025-05-10T13:53:32.572610Z","shell.execute_reply":"2025-05-10T13:53:32.585405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels[\"resid\"].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:32.586810Z","iopub.execute_input":"2025-05-10T13:53:32.587042Z","iopub.status.idle":"2025-05-10T13:53:32.605935Z","shell.execute_reply.started":"2025-05-10T13:53:32.587023Z","shell.execute_reply":"2025-05-10T13:53:32.605312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"px.box(train_labels[\"resid\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:32.608329Z","iopub.execute_input":"2025-05-10T13:53:32.608539Z","iopub.status.idle":"2025-05-10T13:53:34.395107Z","shell.execute_reply.started":"2025-05-10T13:53:32.608523Z","shell.execute_reply":"2025-05-10T13:53:34.393265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sequence.loc[train_sequence[\"target_id\"]==\"4V6X_A5\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.396493Z","iopub.execute_input":"2025-05-10T13:53:34.396734Z","iopub.status.idle":"2025-05-10T13:53:34.420331Z","shell.execute_reply.started":"2025-05-10T13:53:34.396713Z","shell.execute_reply":"2025-05-10T13:53:34.419345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"one_seq=train_sequence.iloc[(1,1)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.421375Z","iopub.execute_input":"2025-05-10T13:53:34.421650Z","iopub.status.idle":"2025-05-10T13:53:34.433774Z","shell.execute_reply.started":"2025-05-10T13:53:34.421630Z","shell.execute_reply":"2025-05-10T13:53:34.432842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"one_seq","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.434714Z","iopub.execute_input":"2025-05-10T13:53:34.435017Z","iopub.status.idle":"2025-05-10T13:53:34.449883Z","shell.execute_reply.started":"2025-05-10T13:53:34.434966Z","shell.execute_reply":"2025-05-10T13:53:34.449295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna=train_sequence[[\"target_id\",\"sequence\"]].copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.450655Z","iopub.execute_input":"2025-05-10T13:53:34.450931Z","iopub.status.idle":"2025-05-10T13:53:34.468142Z","shell.execute_reply.started":"2025-05-10T13:53:34.450901Z","shell.execute_reply":"2025-05-10T13:53:34.467502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.468885Z","iopub.execute_input":"2025-05-10T13:53:34.469195Z","iopub.status.idle":"2025-05-10T13:53:34.612574Z","shell.execute_reply.started":"2025-05-10T13:53:34.469166Z","shell.execute_reply":"2025-05-10T13:53:34.611757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna.dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.613533Z","iopub.execute_input":"2025-05-10T13:53:34.613806Z","iopub.status.idle":"2025-05-10T13:53:34.630076Z","shell.execute_reply.started":"2025-05-10T13:53:34.613776Z","shell.execute_reply":"2025-05-10T13:53:34.629263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#PAD_TOKEN = '<PAD>'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.630942Z","iopub.execute_input":"2025-05-10T13:53:34.631249Z","iopub.status.idle":"2025-05-10T13:53:34.643662Z","shell.execute_reply.started":"2025-05-10T13:53:34.631220Z","shell.execute_reply":"2025-05-10T13:53:34.642809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#rna.loc['sequence']=rna['sequence'].apply(lambda x:str(x)+' ')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.644367Z","iopub.execute_input":"2025-05-10T13:53:34.644617Z","iopub.status.idle":"2025-05-10T13:53:34.661557Z","shell.execute_reply.started":"2025-05-10T13:53:34.644598Z","shell.execute_reply":"2025-05-10T13:53:34.660774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna['sequence'].iloc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.662370Z","iopub.execute_input":"2025-05-10T13:53:34.662617Z","iopub.status.idle":"2025-05-10T13:53:34.680597Z","shell.execute_reply.started":"2025-05-10T13:53:34.662588Z","shell.execute_reply":"2025-05-10T13:53:34.679690Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.681459Z","iopub.execute_input":"2025-05-10T13:53:34.681716Z","iopub.status.idle":"2025-05-10T13:53:34.699717Z","shell.execute_reply.started":"2025-05-10T13:53:34.681698Z","shell.execute_reply":"2025-05-10T13:53:34.698842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_nuc =set()\nfor nuc in one_seq:\n    unique_nuc.add(nuc)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.700450Z","iopub.execute_input":"2025-05-10T13:53:34.700628Z","iopub.status.idle":"2025-05-10T13:53:34.715484Z","shell.execute_reply.started":"2025-05-10T13:53:34.700612Z","shell.execute_reply":"2025-05-10T13:53:34.714799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_nuc=sorted(unique_nuc)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.716286Z","iopub.execute_input":"2025-05-10T13:53:34.716563Z","iopub.status.idle":"2025-05-10T13:53:34.731453Z","shell.execute_reply.started":"2025-05-10T13:53:34.716536Z","shell.execute_reply":"2025-05-10T13:53:34.730628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_nuc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.732298Z","iopub.execute_input":"2025-05-10T13:53:34.732549Z","iopub.status.idle":"2025-05-10T13:53:34.748272Z","shell.execute_reply.started":"2025-05-10T13:53:34.732524Z","shell.execute_reply":"2025-05-10T13:53:34.747475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"one_seq_chars=[char for char in one_seq]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.748932Z","iopub.execute_input":"2025-05-10T13:53:34.749163Z","iopub.status.idle":"2025-05-10T13:53:34.762817Z","shell.execute_reply.started":"2025-05-10T13:53:34.749145Z","shell.execute_reply":"2025-05-10T13:53:34.762264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"one_seq_chars","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.767021Z","iopub.execute_input":"2025-05-10T13:53:34.767230Z","iopub.status.idle":"2025-05-10T13:53:34.783475Z","shell.execute_reply.started":"2025-05-10T13:53:34.767212Z","shell.execute_reply":"2025-05-10T13:53:34.782618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"char_to_idx ={char: idx+1 for idx, char in enumerate(unique_nuc)}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.785565Z","iopub.execute_input":"2025-05-10T13:53:34.785769Z","iopub.status.idle":"2025-05-10T13:53:34.798499Z","shell.execute_reply.started":"2025-05-10T13:53:34.785752Z","shell.execute_reply":"2025-05-10T13:53:34.797845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"char_to_idx","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.799460Z","iopub.execute_input":"2025-05-10T13:53:34.799731Z","iopub.status.idle":"2025-05-10T13:53:34.815003Z","shell.execute_reply.started":"2025-05-10T13:53:34.799705Z","shell.execute_reply":"2025-05-10T13:53:34.814315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"char_to_idx[\" \"] = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.815716Z","iopub.execute_input":"2025-05-10T13:53:34.815898Z","iopub.status.idle":"2025-05-10T13:53:34.830347Z","shell.execute_reply.started":"2025-05-10T13:53:34.815881Z","shell.execute_reply":"2025-05-10T13:53:34.829489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"char_to_idx","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.831062Z","iopub.execute_input":"2025-05-10T13:53:34.831277Z","iopub.status.idle":"2025-05-10T13:53:34.847161Z","shell.execute_reply.started":"2025-05-10T13:53:34.831260Z","shell.execute_reply":"2025-05-10T13:53:34.846562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_text(text, max_length):\n    encoded = [char_to_idx.get(ch, 0) for ch in text]  # Convert chars to IDs\n    return encoded[:max_length] + [0] * (max_length - len(encoded))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.847911Z","iopub.execute_input":"2025-05-10T13:53:34.848196Z","iopub.status.idle":"2025-05-10T13:53:34.861632Z","shell.execute_reply.started":"2025-05-10T13:53:34.848169Z","shell.execute_reply":"2025-05-10T13:53:34.861001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna['num_seq']=rna['sequence'].apply(lambda x: encode_text(x, maxlength))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.862398Z","iopub.execute_input":"2025-05-10T13:53:34.862619Z","iopub.status.idle":"2025-05-10T13:53:34.926835Z","shell.execute_reply.started":"2025-05-10T13:53:34.862600Z","shell.execute_reply":"2025-05-10T13:53:34.926229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna['num_seq']=rna['num_seq'].apply(lambda x: np.array(x))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:34.927496Z","iopub.execute_input":"2025-05-10T13:53:34.927693Z","iopub.status.idle":"2025-05-10T13:53:35.158638Z","shell.execute_reply.started":"2025-05-10T13:53:34.927676Z","shell.execute_reply":"2025-05-10T13:53:35.157970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(rna.iloc[0,2])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.159376Z","iopub.execute_input":"2025-05-10T13:53:35.159583Z","iopub.status.idle":"2025-05-10T13:53:35.164811Z","shell.execute_reply.started":"2025-05-10T13:53:35.159565Z","shell.execute_reply":"2025-05-10T13:53:35.163817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#len(rna.iloc[1,3])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.165672Z","iopub.execute_input":"2025-05-10T13:53:35.165939Z","iopub.status.idle":"2025-05-10T13:53:35.180043Z","shell.execute_reply.started":"2025-05-10T13:53:35.165912Z","shell.execute_reply":"2025-05-10T13:53:35.179329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gx=train_labels[train_labels['resname']=='G']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.180823Z","iopub.execute_input":"2025-05-10T13:53:35.181115Z","iopub.status.idle":"2025-05-10T13:53:35.208226Z","shell.execute_reply.started":"2025-05-10T13:53:35.181088Z","shell.execute_reply":"2025-05-10T13:53:35.207537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gx['x_1'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.209142Z","iopub.execute_input":"2025-05-10T13:53:35.209446Z","iopub.status.idle":"2025-05-10T13:53:35.234588Z","shell.execute_reply.started":"2025-05-10T13:53:35.209419Z","shell.execute_reply":"2025-05-10T13:53:35.233858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"px.histogram(gx['x_1'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.235348Z","iopub.execute_input":"2025-05-10T13:53:35.235585Z","iopub.status.idle":"2025-05-10T13:53:35.339436Z","shell.execute_reply.started":"2025-05-10T13:53:35.235567Z","shell.execute_reply":"2025-05-10T13:53:35.338438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.340443Z","iopub.execute_input":"2025-05-10T13:53:35.340689Z","iopub.status.idle":"2025-05-10T13:53:35.362454Z","shell.execute_reply.started":"2025-05-10T13:53:35.340668Z","shell.execute_reply":"2025-05-10T13:53:35.361567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels=train_labels.dropna()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.363523Z","iopub.execute_input":"2025-05-10T13:53:35.363839Z","iopub.status.idle":"2025-05-10T13:53:35.392694Z","shell.execute_reply.started":"2025-05-10T13:53:35.363815Z","shell.execute_reply":"2025-05-10T13:53:35.392010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.393570Z","iopub.execute_input":"2025-05-10T13:53:35.393821Z","iopub.status.idle":"2025-05-10T13:53:35.405162Z","shell.execute_reply.started":"2025-05-10T13:53:35.393790Z","shell.execute_reply":"2025-05-10T13:53:35.404518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols=['x_1','y_1','z_1']\nremaining_cols=['ID','resname','resid']\nminmax_scaler = MinMaxScaler()\n#minmax = minmax_scaler.fit_transform(train_labels[cols])\n\nstandard_scaler = StandardScaler()\n#data_standardized = standard_scaler.fit_transform(train_labels[cols])\n\npreprocessor = ColumnTransformer(\n    transformers=[\n        #('std', StandardScaler(), cols),\n        ('minmax', MinMaxScaler(), cols),\n        ('passthrough', 'passthrough', remaining_cols)\n    ]\n)\n\nxyz_std=preprocessor.fit_transform(train_labels)\n\ntrain_labels_norm = pd.DataFrame(xyz_std, columns=cols+remaining_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.405960Z","iopub.execute_input":"2025-05-10T13:53:35.406297Z","iopub.status.idle":"2025-05-10T13:53:35.462854Z","shell.execute_reply.started":"2025-05-10T13:53:35.406266Z","shell.execute_reply":"2025-05-10T13:53:35.462209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels_norm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.463601Z","iopub.execute_input":"2025-05-10T13:53:35.463833Z","iopub.status.idle":"2025-05-10T13:53:35.473698Z","shell.execute_reply.started":"2025-05-10T13:53:35.463813Z","shell.execute_reply":"2025-05-10T13:53:35.473080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"renc=[]\nfor i,row in rna.iterrows():\n    #print(i)\n    temp_label=train_labels[train_labels['ID'].str.startswith( row['target_id'])]\n    np_temp=temp_label[['x_1','y_1','z_1']].to_numpy()\n    padding = np.zeros((4300 - len(np_temp),np_temp.shape[1]), dtype=np_temp.dtype)\n    #print(np_temp)\n    renc.append(np.concatenate((np_temp, padding)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:53:35.474460Z","iopub.execute_input":"2025-05-10T13:53:35.474735Z","iopub.status.idle":"2025-05-10T13:54:00.044499Z","shell.execute_reply.started":"2025-05-10T13:53:35.474708Z","shell.execute_reply":"2025-05-10T13:54:00.043570Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"renc[0].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.045440Z","iopub.execute_input":"2025-05-10T13:54:00.045743Z","iopub.status.idle":"2025-05-10T13:54:00.051263Z","shell.execute_reply.started":"2025-05-10T13:54:00.045715Z","shell.execute_reply":"2025-05-10T13:54:00.050367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#renc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.052117Z","iopub.execute_input":"2025-05-10T13:54:00.052461Z","iopub.status.idle":"2025-05-10T13:54:00.068070Z","shell.execute_reply.started":"2025-05-10T13:54:00.052434Z","shell.execute_reply":"2025-05-10T13:54:00.067339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna['encoded'] = renc    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.068790Z","iopub.execute_input":"2025-05-10T13:54:00.068970Z","iopub.status.idle":"2025-05-10T13:54:00.082113Z","shell.execute_reply.started":"2025-05-10T13:54:00.068955Z","shell.execute_reply":"2025-05-10T13:54:00.081453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp_label=train_labels[train_labels['ID'].str.startswith( row['target_id'])]\nnp_temp=temp_label[['x_1','y_1','z_1']].to_numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.082823Z","iopub.execute_input":"2025-05-10T13:54:00.083083Z","iopub.status.idle":"2025-05-10T13:54:00.124816Z","shell.execute_reply.started":"2025-05-10T13:54:00.083060Z","shell.execute_reply":"2025-05-10T13:54:00.124040Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#train_labels[train_labels['ID']== '1SCL_A']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.125734Z","iopub.execute_input":"2025-05-10T13:54:00.126044Z","iopub.status.idle":"2025-05-10T13:54:00.129207Z","shell.execute_reply.started":"2025-05-10T13:54:00.126009Z","shell.execute_reply":"2025-05-10T13:54:00.128450Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np_temp.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.129928Z","iopub.execute_input":"2025-05-10T13:54:00.130142Z","iopub.status.idle":"2025-05-10T13:54:00.144216Z","shell.execute_reply.started":"2025-05-10T13:54:00.130104Z","shell.execute_reply":"2025-05-10T13:54:00.143459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.144913Z","iopub.execute_input":"2025-05-10T13:54:00.145145Z","iopub.status.idle":"2025-05-10T13:54:00.359460Z","shell.execute_reply.started":"2025-05-10T13:54:00.145127Z","shell.execute_reply":"2025-05-10T13:54:00.358533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.360499Z","iopub.execute_input":"2025-05-10T13:54:00.360765Z","iopub.status.idle":"2025-05-10T13:54:00.366680Z","shell.execute_reply.started":"2025-05-10T13:54:00.360744Z","shell.execute_reply":"2025-05-10T13:54:00.365809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna.iloc[1,2]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.367572Z","iopub.execute_input":"2025-05-10T13:54:00.367864Z","iopub.status.idle":"2025-05-10T13:54:00.382409Z","shell.execute_reply.started":"2025-05-10T13:54:00.367836Z","shell.execute_reply":"2025-05-10T13:54:00.381648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(rna)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.383181Z","iopub.execute_input":"2025-05-10T13:54:00.383403Z","iopub.status.idle":"2025-05-10T13:54:00.397688Z","shell.execute_reply.started":"2025-05-10T13:54:00.383385Z","shell.execute_reply":"2025-05-10T13:54:00.396863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna.shape[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.398477Z","iopub.execute_input":"2025-05-10T13:54:00.398728Z","iopub.status.idle":"2025-05-10T13:54:00.413180Z","shell.execute_reply.started":"2025-05-10T13:54:00.398703Z","shell.execute_reply":"2025-05-10T13:54:00.412371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#npchar_index=np.array(list(char_to_idx.keys())).reshape(-1, 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.413933Z","iopub.execute_input":"2025-05-10T13:54:00.414155Z","iopub.status.idle":"2025-05-10T13:54:00.426088Z","shell.execute_reply.started":"2025-05-10T13:54:00.414133Z","shell.execute_reply":"2025-05-10T13:54:00.425374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#npchar_index","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.426751Z","iopub.execute_input":"2025-05-10T13:54:00.426946Z","iopub.status.idle":"2025-05-10T13:54:00.438856Z","shell.execute_reply.started":"2025-05-10T13:54:00.426929Z","shell.execute_reply":"2025-05-10T13:54:00.438055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#enc=OneHotEncoder()\n#enc.fit_transform(npchar_index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.439618Z","iopub.execute_input":"2025-05-10T13:54:00.439805Z","iopub.status.idle":"2025-05-10T13:54:00.454121Z","shell.execute_reply.started":"2025-05-10T13:54:00.439788Z","shell.execute_reply":"2025-05-10T13:54:00.453399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#enc.categories_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.454807Z","iopub.execute_input":"2025-05-10T13:54:00.455013Z","iopub.status.idle":"2025-05-10T13:54:00.469466Z","shell.execute_reply.started":"2025-05-10T13:54:00.454996Z","shell.execute_reply":"2025-05-10T13:54:00.468645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#np_one_seq=np.array(one_seq_chars).reshape(-1,1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.470299Z","iopub.execute_input":"2025-05-10T13:54:00.470565Z","iopub.status.idle":"2025-05-10T13:54:00.485479Z","shell.execute_reply.started":"2025-05-10T13:54:00.470535Z","shell.execute_reply":"2025-05-10T13:54:00.484728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#np_one_seq","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.486129Z","iopub.execute_input":"2025-05-10T13:54:00.486314Z","iopub.status.idle":"2025-05-10T13:54:00.499647Z","shell.execute_reply.started":"2025-05-10T13:54:00.486297Z","shell.execute_reply":"2025-05-10T13:54:00.499058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#np_one_seq_encoded=enc.transform(np_one_seq)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.500364Z","iopub.execute_input":"2025-05-10T13:54:00.500589Z","iopub.status.idle":"2025-05-10T13:54:00.514325Z","shell.execute_reply.started":"2025-05-10T13:54:00.500560Z","shell.execute_reply":"2025-05-10T13:54:00.513629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#np_one_seq_encoded.toarray()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.515218Z","iopub.execute_input":"2025-05-10T13:54:00.515477Z","iopub.status.idle":"2025-05-10T13:54:00.528127Z","shell.execute_reply.started":"2025-05-10T13:54:00.515456Z","shell.execute_reply":"2025-05-10T13:54:00.527316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#position = np.arange(4500)[:, np.newaxis]\n#div_term = np.exp(np.arange(0, 4, 2) * (-np.log(10000.0) / 4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.528922Z","iopub.execute_input":"2025-05-10T13:54:00.529204Z","iopub.status.idle":"2025-05-10T13:54:00.541778Z","shell.execute_reply.started":"2025-05-10T13:54:00.529174Z","shell.execute_reply":"2025-05-10T13:54:00.540995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Seq2SeqTransformer(nn.Module):\n    def __init__(self, vocab_size, d_model=3, nhead=3, num_layers=3):\n        super().__init__()\n        self.d_model = d_model\n        \n        # Encoder\n        self.token_embed = nn.Embedding(vocab_size, d_model)\n        self.pos_embed = nn.Embedding(MAX_LENGTH, d_model)\n        self.encoder_layers = nn.TransformerEncoderLayer(d_model, nhead)\n        self.encoder = nn.TransformerEncoder(self.encoder_layers, num_layers)\n        #linear1 = nn.Linear(3, 64)\n        \n        # Decoder\n        self.decoder_layers = nn.TransformerDecoderLayer(d_model, nhead)\n        self.decoder = nn.TransformerDecoder(self.decoder_layers, num_layers)\n        self.fc_out = nn.Linear(d_model, vocab_size)\n\n    def encode(self, src):\n        positions = torch.arange(src.size(1), device=src.device)\n        tok_emb = self.token_embed(src) * np.sqrt(self.d_model)\n        #print(tok_emb)\n        pos_emb = self.pos_embed(positions)\n        #print(pos_emb)\n        encoded = self.encoder(tok_emb + pos_emb)\n        #print(encoded)\n        return encoded\n\n    def decode(self, mem):\n        tgt=tgt.float()\n        batch_size, tgt_seq_len,_ = tgt.size()\n        tgt=tgt.reshape(4,-1)\n        #print(tgt.shape)\n        L1=nn.Linear(4300*3,4300*2)\n        L2=nn.Linear(4300*2,4300)\n        #print(tgt.dtype)\n        tgt2=L1(tgt)\n        #print(tgt2.dtype)\n        tgt3=L2(tgt2)\n        #print(tgt3.shape)\n\n    def forward(self, src):\n        enc = self.encode(src)\n        #outputs = self.decode(memory, tgt)\n        return enc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.542597Z","iopub.execute_input":"2025-05-10T13:54:00.542798Z","iopub.status.idle":"2025-05-10T13:54:00.553427Z","shell.execute_reply.started":"2025-05-10T13:54:00.542772Z","shell.execute_reply":"2025-05-10T13:54:00.552407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SequenceDataset(Dataset):\n    def __init__(self, rna, max_length):\n        self.max_length = max_length\n        self.num_enc= rna['num_seq'].to_numpy()\n        self.nuc_enc= rna['encoded'].to_numpy()\n        #col_mean = np.nanmean(self.num_enc, axis=0)\n        #self.num_enc = np.where(np.isnan(self.num_enc), 0, self.num_enc)\n        #col_mean = np.nanmean(self.nuc_enc, axis=0)\n        #self.nuc_enc = np.where(np.isnan(self.nuc_enc), 0, self.nuc_enc)\n    \n    def __len__(self): return rna.shape[0]\n        \n    def __getitem__(self, idx): return self.num_enc[idx], self.nuc_enc[idx]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.554311Z","iopub.execute_input":"2025-05-10T13:54:00.554579Z","iopub.status.idle":"2025-05-10T13:54:00.571926Z","shell.execute_reply.started":"2025-05-10T13:54:00.554553Z","shell.execute_reply":"2025-05-10T13:54:00.571171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna.shape[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.572687Z","iopub.execute_input":"2025-05-10T13:54:00.572888Z","iopub.status.idle":"2025-05-10T13:54:00.592213Z","shell.execute_reply.started":"2025-05-10T13:54:00.572871Z","shell.execute_reply":"2025-05-10T13:54:00.591432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_enc= rna['num_seq'].to_numpy()\nnuc_enc= rna['encoded'].to_numpy().reshape(-1,1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.593024Z","iopub.execute_input":"2025-05-10T13:54:00.593263Z","iopub.status.idle":"2025-05-10T13:54:00.608265Z","shell.execute_reply.started":"2025-05-10T13:54:00.593245Z","shell.execute_reply":"2025-05-10T13:54:00.607474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_enc[0].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.609038Z","iopub.execute_input":"2025-05-10T13:54:00.609532Z","iopub.status.idle":"2025-05-10T13:54:00.623525Z","shell.execute_reply.started":"2025-05-10T13:54:00.609512Z","shell.execute_reply":"2025-05-10T13:54:00.622749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nuc_enc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.624443Z","iopub.execute_input":"2025-05-10T13:54:00.624717Z","iopub.status.idle":"2025-05-10T13:54:00.638382Z","shell.execute_reply.started":"2025-05-10T13:54:00.624689Z","shell.execute_reply":"2025-05-10T13:54:00.637590Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nuc_enc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.644887Z","iopub.execute_input":"2025-05-10T13:54:00.645116Z","iopub.status.idle":"2025-05-10T13:54:00.654145Z","shell.execute_reply.started":"2025-05-10T13:54:00.645098Z","shell.execute_reply":"2025-05-10T13:54:00.653458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_enc.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.655911Z","iopub.execute_input":"2025-05-10T13:54:00.656154Z","iopub.status.idle":"2025-05-10T13:54:00.669073Z","shell.execute_reply.started":"2025-05-10T13:54:00.656136Z","shell.execute_reply":"2025-05-10T13:54:00.668480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.669912Z","iopub.execute_input":"2025-05-10T13:54:00.670192Z","iopub.status.idle":"2025-05-10T13:54:00.734828Z","shell.execute_reply.started":"2025-05-10T13:54:00.670173Z","shell.execute_reply":"2025-05-10T13:54:00.734049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Hyperparameters\nMAX_LENGTH = 4300\nBATCH_SIZE = 4\nVOCAB_SIZE = 5\n\n# Initialize\nmodel = Seq2SeqTransformer(VOCAB_SIZE)\n#criterion = nn.CrossEntropyLoss(ignore_index=4)# Ignore padding\ncriterion = nn.MSELoss()\n#criterion = nn.CosineSimilarity()\noptimizer = optim.Adam(model.parameters(), lr=1e-3)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:00.735822Z","iopub.execute_input":"2025-05-10T13:54:00.736155Z","iopub.status.idle":"2025-05-10T13:54:02.631869Z","shell.execute_reply.started":"2025-05-10T13:54:00.736124Z","shell.execute_reply":"2025-05-10T13:54:02.630961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model=model.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.632797Z","iopub.execute_input":"2025-05-10T13:54:02.633413Z","iopub.status.idle":"2025-05-10T13:54:02.826902Z","shell.execute_reply.started":"2025-05-10T13:54:02.633381Z","shell.execute_reply":"2025-05-10T13:54:02.826261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = SequenceDataset(rna, MAX_LENGTH)\ndataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.827688Z","iopub.execute_input":"2025-05-10T13:54:02.828001Z","iopub.status.idle":"2025-05-10T13:54:02.832171Z","shell.execute_reply.started":"2025-05-10T13:54:02.827955Z","shell.execute_reply":"2025-05-10T13:54:02.831447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch=next(iter(dataloader))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.833061Z","iopub.execute_input":"2025-05-10T13:54:02.833321Z","iopub.status.idle":"2025-05-10T13:54:02.872921Z","shell.execute_reply.started":"2025-05-10T13:54:02.833291Z","shell.execute_reply":"2025-05-10T13:54:02.872133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch[0].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.873642Z","iopub.execute_input":"2025-05-10T13:54:02.873830Z","iopub.status.idle":"2025-05-10T13:54:02.878344Z","shell.execute_reply.started":"2025-05-10T13:54:02.873813Z","shell.execute_reply":"2025-05-10T13:54:02.877663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch[1].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.879125Z","iopub.execute_input":"2025-05-10T13:54:02.879342Z","iopub.status.idle":"2025-05-10T13:54:02.892488Z","shell.execute_reply.started":"2025-05-10T13:54:02.879312Z","shell.execute_reply":"2025-05-10T13:54:02.891806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch[0].size(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.893306Z","iopub.execute_input":"2025-05-10T13:54:02.893577Z","iopub.status.idle":"2025-05-10T13:54:02.908433Z","shell.execute_reply.started":"2025-05-10T13:54:02.893550Z","shell.execute_reply":"2025-05-10T13:54:02.907780Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna_seq_only=pd.DataFrame(rna['sequence'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.909144Z","iopub.execute_input":"2025-05-10T13:54:02.909387Z","iopub.status.idle":"2025-05-10T13:54:02.923066Z","shell.execute_reply.started":"2025-05-10T13:54:02.909366Z","shell.execute_reply":"2025-05-10T13:54:02.922355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rna_seq_only","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.923802Z","iopub.execute_input":"2025-05-10T13:54:02.924064Z","iopub.status.idle":"2025-05-10T13:54:02.941580Z","shell.execute_reply.started":"2025-05-10T13:54:02.924044Z","shell.execute_reply":"2025-05-10T13:54:02.940817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"d_len=len(dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.942275Z","iopub.execute_input":"2025-05-10T13:54:02.942460Z","iopub.status.idle":"2025-05-10T13:54:02.954782Z","shell.execute_reply.started":"2025-05-10T13:54:02.942444Z","shell.execute_reply":"2025-05-10T13:54:02.953957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ratio = 0.85\nvalidation_ratio = 0.15\n\n# Step 4: Calculate the sizes for each split\ndataset_size = len(dataset)\ntrain_size = int(train_ratio * dataset_size)\nvalidation_size = dataset_size - train_size","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.955594Z","iopub.execute_input":"2025-05-10T13:54:02.955844Z","iopub.status.idle":"2025-05-10T13:54:02.969616Z","shell.execute_reply.started":"2025-05-10T13:54:02.955826Z","shell.execute_reply":"2025-05-10T13:54:02.968930Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset, validation_dataset = random_split(dataset, [train_size, validation_size])\n\ntrain_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\nvalidation_loader = DataLoader(validation_dataset, batch_size=8, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.970399Z","iopub.execute_input":"2025-05-10T13:54:02.970583Z","iopub.status.idle":"2025-05-10T13:54:02.985451Z","shell.execute_reply.started":"2025-05-10T13:54:02.970567Z","shell.execute_reply":"2025-05-10T13:54:02.984788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'Total dataset size: {dataset_size}')\nprint(f'Training dataset size: {len(train_dataset)}')\nprint(f'Validation dataset size: {len(validation_dataset)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:02.986129Z","iopub.execute_input":"2025-05-10T13:54:02.986360Z","iopub.status.idle":"2025-05-10T13:54:03.002653Z","shell.execute_reply.started":"2025-05-10T13:54:02.986331Z","shell.execute_reply":"2025-05-10T13:54:03.002045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:03.003427Z","iopub.execute_input":"2025-05-10T13:54:03.003682Z","iopub.status.idle":"2025-05-10T13:54:03.019615Z","shell.execute_reply.started":"2025-05-10T13:54:03.003653Z","shell.execute_reply":"2025-05-10T13:54:03.019002Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:03.020264Z","iopub.execute_input":"2025-05-10T13:54:03.020446Z","iopub.status.idle":"2025-05-10T13:54:03.034424Z","shell.execute_reply.started":"2025-05-10T13:54:03.020430Z","shell.execute_reply":"2025-05-10T13:54:03.033685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:03.035042Z","iopub.execute_input":"2025-05-10T13:54:03.035350Z","iopub.status.idle":"2025-05-10T13:54:03.048870Z","shell.execute_reply.started":"2025-05-10T13:54:03.035326Z","shell.execute_reply":"2025-05-10T13:54:03.048197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"epochs=100\nmodel.train()\nfor epoch in range(epochs):\n    total_loss = 0\n    train_bar = tqdm(train_loader, desc=f\"Training Epoch {epoch+1}/{epochs}\")\n    for src, tgt in train_bar:\n        src, tgt=src.to(device), tgt.to(device)\n        #src= src.to(device)\n        #src = src.float()\n        tgt = tgt.float()\n        optimizer.zero_grad()\n        #print(src)\n        #print(tgt.dtype)\n        \n        # Shift target for teacher forcing\n        \n        output = model(src)\n        #print(output.dtype)\n        #print(tgt.dtype)\n        #outputscpu=outputs.to('cpu')\n        loss = criterion(output, tgt)\n        #torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        \n        loss.backward()\n        #torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n        optimizer.step()\n        \n        total_loss += loss.item()\n    \n    avg_loss = total_loss / len(dataloader)\n    scheduler.step(avg_loss)\n    print(f'Epoch {epoch+1}, Loss: {avg_loss:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T13:54:03.049612Z","iopub.execute_input":"2025-05-10T13:54:03.049886Z","iopub.status.idle":"2025-05-10T14:03:52.452957Z","shell.execute_reply.started":"2025-05-10T13:54:03.049859Z","shell.execute_reply":"2025-05-10T14:03:52.452184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tgt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:52.453767Z","iopub.execute_input":"2025-05-10T14:03:52.454041Z","iopub.status.idle":"2025-05-10T14:03:52.842753Z","shell.execute_reply.started":"2025-05-10T14:03:52.454018Z","shell.execute_reply":"2025-05-10T14:03:52.841817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#tgt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:52.843651Z","iopub.execute_input":"2025-05-10T14:03:52.843967Z","iopub.status.idle":"2025-05-10T14:03:52.847427Z","shell.execute_reply.started":"2025-05-10T14:03:52.843937Z","shell.execute_reply":"2025-05-10T14:03:52.846446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#loss=criterion(output, tgt)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:52.848106Z","iopub.execute_input":"2025-05-10T14:03:52.848322Z","iopub.status.idle":"2025-05-10T14:03:52.861203Z","shell.execute_reply.started":"2025-05-10T14:03:52.848304Z","shell.execute_reply":"2025-05-10T14:03:52.860444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:52.862036Z","iopub.execute_input":"2025-05-10T14:03:52.862317Z","iopub.status.idle":"2025-05-10T14:03:52.875416Z","shell.execute_reply.started":"2025-05-10T14:03:52.862289Z","shell.execute_reply":"2025-05-10T14:03:52.874703Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#train(model, dataloader, epochs=10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:52.876111Z","iopub.execute_input":"2025-05-10T14:03:52.876328Z","iopub.status.idle":"2025-05-10T14:03:52.890119Z","shell.execute_reply.started":"2025-05-10T14:03:52.876310Z","shell.execute_reply":"2025-05-10T14:03:52.889416Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"for src, tgt in dataloader:\n    optimizer.zero_grad()\n    print(src.shape)\n    #tgt_input = tgt[:, :-1]\n    #tgt_output = tgt[:, 1:]\n    print(tgt.shape)\n    #print(tgt[:, :-1].shape)\n    #print(tgt[:, 1:].shape)\n\n    # Encoder\n    token_embed = nn.Embedding(5, 3)\n    pos_embed = nn.Embedding(4300, 3)\n    encoder_layers = nn.TransformerEncoderLayer(3, 3)\n    encoder = nn.TransformerEncoder(encoder_layers, 3)\n    #linear1 = nn.Linear(3, 64)\n    print(src.size(1))\n    positions = torch.arange(src.size(1), device=src.device)\n    tok_emb = token_embed(src) * np.sqrt(3)\n    print(tok_emb.shape)\n    pos_emb = pos_embed(positions)\n    print(pos_emb.shape)\n    encoded = encoder(tok_emb + pos_emb)\n    print(\"##\",encoded.shape)\n    loss = criterion(encoded, tgt)\n    print(\"loss\", loss)\n    #print(\"##\",encoded)\n    # Decoder\n    tgt=tgt.float()\n    batch_size, tgt_seq_len,_ = tgt.size()\n    tgt=tgt.reshape(4,-1)\n    print(tgt.shape)\n    L1=nn.Linear(4300*3,4300*2)\n    L2=nn.Linear(4300*2,4300)\n    print(tgt.dtype)\n    tgt2=L1(tgt)\n    print(tgt2.dtype)\n    tgt3=L2(tgt2)\n    print(tgt3.shape)\n    #tok_emb = token_embed(tgt) * np.sqrt(3)  # (batch_size, tgt_seq_len, d_model)\n    #positions = torch.arange(tgt_seq_len, device=tgt.device)\n    #pos_emb = pos_embed(positions)\n    #pos_emb = pos_emb.unsqueeze(0).expand(4, tgt_seq_len, 3)\n\n    # Add embeddings\n    #tgt_emb = tok_emb + pos_emb  # (batch_size, tgt_seq_len, d_model)\n\n    # Permute for Transformer (seq_len, batch_size, d_model)\n    #tgt_emb = tgt_emb.permute(1, 0, 2)    # (tgt_seq_len, batch_size, d_model)\n    #memory = memory.permute(1, 0, 2)      # (src_seq_len, batch_size, d_model)\n\n    # Pass through decoder\n    #decoded = self.decoder(tgt_emb, memory)  # (tgt_seq_len, batch_size, d_model)\n\n    # Permute back\n    #decoded = decoded.permute(1, 0, 2)  # (batch_size, tgt_seq_len, d_model)\n\n    # Final output projection\n    #output = self.output_layer(decoded)\n\n\n    #decoder_layers = nn.TransformerDecoderLayer(3, 3)\n    #decoder = nn.TransformerDecoder(decoder_layers, 3)\n    #fc_out = nn.Linear(3, 3)\n\n\n    #positions = torch.arange(tgt.size(1), device=tgt.device)\n    print(positions)\n    #projected = linear_in(tgt)\n    #tok_emb = token_embed(src) * np.sqrt(3)\n    #pos_emb = pos_embed(positions)\n    #decoded = decoder(tok_emb + pos_emb, memory)\n    break\n    ","metadata":{"execution":{"iopub.status.busy":"2025-05-10T13:41:01.784823Z","iopub.execute_input":"2025-05-10T13:41:01.785102Z","iopub.status.idle":"2025-05-10T13:41:04.363951Z","shell.execute_reply.started":"2025-05-10T13:41:01.785063Z","shell.execute_reply":"2025-05-10T13:41:04.363031Z"},"_kg_hide-input":true,"_kg_hide-output":true}},{"cell_type":"code","source":"#tgt_input","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:52.890785Z","iopub.execute_input":"2025-05-10T14:03:52.890968Z","iopub.status.idle":"2025-05-10T14:03:52.905402Z","shell.execute_reply.started":"2025-05-10T14:03:52.890951Z","shell.execute_reply":"2025-05-10T14:03:52.904662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_outs=[]\ntestlabels=[]\ntest_loss=[]\nfor inputs, tar in validation_loader:\n    inputs, tar=inputs.to(device), tar.to(device)\n    test_outputs = model(inputs)\n    output = model(src)\n    loss = criterion(output, tgt)\n    test_loss.append( loss.item())\n    outputscpu=test_outputs.to('cpu')\n    nptestout=outputscpu.detach()\n    nptestout=nptestout.numpy()\n    for ele in nptestout:\n        #print(nptestout.shape)\n        test_outs.append(ele)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:13:21.399899Z","iopub.execute_input":"2025-05-10T14:13:21.400263Z","iopub.status.idle":"2025-05-10T14:13:22.125015Z","shell.execute_reply.started":"2025-05-10T14:13:21.400234Z","shell.execute_reply":"2025-05-10T14:13:22.124230Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:17:11.883062Z","iopub.execute_input":"2025-05-10T14:17:11.883384Z","iopub.status.idle":"2025-05-10T14:17:11.888425Z","shell.execute_reply.started":"2025-05-10T14:17:11.883359Z","shell.execute_reply":"2025-05-10T14:17:11.887489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(test_outs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:53.345646Z","iopub.execute_input":"2025-05-10T14:03:53.345953Z","iopub.status.idle":"2025-05-10T14:03:53.350712Z","shell.execute_reply.started":"2025-05-10T14:03:53.345924Z","shell.execute_reply":"2025-05-10T14:03:53.350060Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_outs[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:53.351451Z","iopub.execute_input":"2025-05-10T14:03:53.351735Z","iopub.status.idle":"2025-05-10T14:03:53.364786Z","shell.execute_reply.started":"2025-05-10T14:03:53.351704Z","shell.execute_reply":"2025-05-10T14:03:53.364058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_outs[0].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:03:53.365442Z","iopub.execute_input":"2025-05-10T14:03:53.365715Z","iopub.status.idle":"2025-05-10T14:03:53.379945Z","shell.execute_reply.started":"2025-05-10T14:03:53.365695Z","shell.execute_reply":"2025-05-10T14:03:53.379171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(test_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T14:12:37.603672Z","iopub.execute_input":"2025-05-10T14:12:37.604039Z","iopub.status.idle":"2025-05-10T14:12:37.609116Z","shell.execute_reply.started":"2025-05-10T14:12:37.604009Z","shell.execute_reply":"2025-05-10T14:12:37.608292Z"}},"outputs":[],"execution_count":null}]}