{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install umap-learn --upgrade","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:01:41.884972Z","iopub.execute_input":"2023-10-17T00:01:41.885371Z","iopub.status.idle":"2023-10-17T00:01:56.176860Z","shell.execute_reply.started":"2023-10-17T00:01:41.885327Z","shell.execute_reply":"2023-10-17T00:01:56.175279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport tensorflow as tf\nimport numpy as np\nimport torch\nimport matplotlib.pyplot as plt\nimport umap\nfrom umap import UMAP\nimport lightgbm as lgb\nfrom lightgbm  import LGBMRegressor as LGBM\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Sequential\nfrom keras.layers import Dense, BatchNormalization, Activation, Dropout\nfrom keras.regularizers import l1_l2\nfrom keras.regularizers import l2\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping\nfrom transformers import RobertaTokenizer, RobertaModel","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:01:56.179059Z","iopub.execute_input":"2023-10-17T00:01:56.179582Z","iopub.status.idle":"2023-10-17T00:02:30.421798Z","shell.execute_reply.started":"2023-10-17T00:01:56.179501Z","shell.execute_reply":"2023-10-17T00:02:30.420590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data ","metadata":{}},{"cell_type":"code","source":"de_train =   pd.read_parquet(\"/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet\")\nid_map = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/id_map.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:30.423110Z","iopub.execute_input":"2023-10-17T00:02:30.423413Z","iopub.status.idle":"2023-10-17T00:02:35.830964Z","shell.execute_reply.started":"2023-10-17T00:02:30.423389Z","shell.execute_reply":"2023-10-17T00:02:35.829977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False:   \n    adata_obs = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/adata_obs_meta.csv')\n    multiome_train = pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/multiome_train.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:35.833946Z","iopub.execute_input":"2023-10-17T00:02:35.834383Z","iopub.status.idle":"2023-10-17T00:02:35.840265Z","shell.execute_reply.started":"2023-10-17T00:02:35.834342Z","shell.execute_reply":"2023-10-17T00:02:35.838815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False:\n    multiome_var = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/multiome_var_meta.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:35.841578Z","iopub.execute_input":"2023-10-17T00:02:35.841918Z","iopub.status.idle":"2023-10-17T00:02:35.862322Z","shell.execute_reply.started":"2023-10-17T00:02:35.841891Z","shell.execute_reply":"2023-10-17T00:02:35.861363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False:\n    multiome_obs = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/multiome_obs_meta.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:35.863631Z","iopub.execute_input":"2023-10-17T00:02:35.864407Z","iopub.status.idle":"2023-10-17T00:02:35.876358Z","shell.execute_reply.started":"2023-10-17T00:02:35.864378Z","shell.execute_reply":"2023-10-17T00:02:35.875174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#adata_train = pl.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/adata_train.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:35.877868Z","iopub.execute_input":"2023-10-17T00:02:35.878238Z","iopub.status.idle":"2023-10-17T00:02:35.892484Z","shell.execute_reply.started":"2023-10-17T00:02:35.878209Z","shell.execute_reply":"2023-10-17T00:02:35.891246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:35.893812Z","iopub.execute_input":"2023-10-17T00:02:35.894108Z","iopub.status.idle":"2023-10-17T00:02:35.937576Z","shell.execute_reply.started":"2023-10-17T00:02:35.894084Z","shell.execute_reply":"2023-10-17T00:02:35.936386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"de_train.duplicated(['cell_type','sm_name']).value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:35.938875Z","iopub.execute_input":"2023-10-17T00:02:35.939180Z","iopub.status.idle":"2023-10-17T00:02:36.685050Z","shell.execute_reply.started":"2023-10-17T00:02:35.939154Z","shell.execute_reply":"2023-10-17T00:02:36.684058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train[de_train.duplicated(subset=['cell_type','sm_name'])]","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:36.688737Z","iopub.execute_input":"2023-10-17T00:02:36.689102Z","iopub.status.idle":"2023-10-17T00:02:36.725665Z","shell.execute_reply.started":"2023-10-17T00:02:36.689075Z","shell.execute_reply":"2023-10-17T00:02:36.724784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Become one with data","metadata":{}},{"cell_type":"code","source":"de_train","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:36.726656Z","iopub.execute_input":"2023-10-17T00:02:36.727368Z","iopub.status.idle":"2023-10-17T00:02:36.757159Z","shell.execute_reply.started":"2023-10-17T00:02:36.727339Z","shell.execute_reply":"2023-10-17T00:02:36.756076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:36.758572Z","iopub.execute_input":"2023-10-17T00:02:36.759170Z","iopub.status.idle":"2023-10-17T00:02:36.769034Z","shell.execute_reply.started":"2023-10-17T00:02:36.759130Z","shell.execute_reply":"2023-10-17T00:02:36.767877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:36.770449Z","iopub.execute_input":"2023-10-17T00:02:36.770860Z","iopub.status.idle":"2023-10-17T00:02:36.813846Z","shell.execute_reply.started":"2023-10-17T00:02:36.770823Z","shell.execute_reply":"2023-10-17T00:02:36.812610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train['cell_type'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:36.815209Z","iopub.execute_input":"2023-10-17T00:02:36.815654Z","iopub.status.idle":"2023-10-17T00:02:36.824870Z","shell.execute_reply.started":"2023-10-17T00:02:36.815612Z","shell.execute_reply":"2023-10-17T00:02:36.823715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess data ","metadata":{}},{"cell_type":"code","source":"MODEL_NAME = \"seyonec/ChemBERTa-zinc-base-v1\"\n#MODEL_NAME = 'seyonec/PubChem10M_SMILES_BPE_450k'\n#MODEL_NAME = 'DeepChem/ChemBERTa-77M-MLM'\n# Talkinizer and model loading\ntokenizer = RobertaTokenizer.from_pretrained(MODEL_NAME)\nmodel = RobertaModel.from_pretrained(MODEL_NAME)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:36.826595Z","iopub.execute_input":"2023-10-17T00:02:36.827164Z","iopub.status.idle":"2023-10-17T00:02:40.844751Z","shell.execute_reply.started":"2023-10-17T00:02:36.827131Z","shell.execute_reply":"2023-10-17T00:02:40.843828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def smiles_to_vector(smiles):\n    # Tokenize SMILES\n    inputs = tokenizer(smiles, return_tensors=\"pt\", truncation=True, padding=\"max_length\", max_length=512)\n\n    with torch.no_grad():\n        outputs = model(**inputs)\n        \n    # Get vector of [CLS] tokens\n    vector = outputs.last_hidden_state[:, 0, :].numpy()\n    vector = outputs.last_hidden_state[0,0, :].numpy()\n    \n    return vector","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:40.846057Z","iopub.execute_input":"2023-10-17T00:02:40.846348Z","iopub.status.idle":"2023-10-17T00:02:40.852427Z","shell.execute_reply.started":"2023-10-17T00:02:40.846323Z","shell.execute_reply":"2023-10-17T00:02:40.851545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smiles_vec_dic= {i:smiles_to_vector(i).flatten() for i in set(de_train.SMILES.to_list())}\nde_train['SMILES_vec'] = de_train['SMILES'].map(smiles_vec_dic)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:02:40.853665Z","iopub.execute_input":"2023-10-17T00:02:40.853921Z","iopub.status.idle":"2023-10-17T00:03:48.677589Z","shell.execute_reply.started":"2023-10-17T00:02:40.853899Z","shell.execute_reply":"2023-10-17T00:03:48.676459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train[['cell_type','sm_name','SMILES_vec']].head()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.679144Z","iopub.execute_input":"2023-10-17T00:03:48.680082Z","iopub.status.idle":"2023-10-17T00:03:48.699755Z","shell.execute_reply.started":"2023-10-17T00:03:48.680044Z","shell.execute_reply":"2023-10-17T00:03:48.698580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train.at[4,'SMILES_vec'].shape","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.701550Z","iopub.execute_input":"2023-10-17T00:03:48.701865Z","iopub.status.idle":"2023-10-17T00:03:48.716077Z","shell.execute_reply.started":"2023-10-17T00:03:48.701840Z","shell.execute_reply":"2023-10-17T00:03:48.714947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shuffle the data\n#de_train = de_train.sample(frac=1.0, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.717477Z","iopub.execute_input":"2023-10-17T00:03:48.717930Z","iopub.status.idle":"2023-10-17T00:03:48.729944Z","shell.execute_reply.started":"2023-10-17T00:03:48.717890Z","shell.execute_reply":"2023-10-17T00:03:48.728852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_columns = [\"cell_type\", \"sm_name\"]\nlabels_columns=[\"cell_type\",\"sm_name\",\"sm_lincs_id\",\"SMILES\",\"control\"]\nlabels = de_train.drop(columns=labels_columns)\nfeatures = pd.DataFrame(de_train, columns=features_columns)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.731576Z","iopub.execute_input":"2023-10-17T00:03:48.732010Z","iopub.status.idle":"2023-10-17T00:03:48.771342Z","shell.execute_reply.started":"2023-10-17T00:03:48.731970Z","shell.execute_reply":"2023-10-17T00:03:48.770417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map_smiles = pd.merge(id_map,de_train[['sm_name','SMILES_vec']].drop_duplicates('sm_name'),on='sm_name',how='left')","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.772659Z","iopub.execute_input":"2023-10-17T00:03:48.773028Z","iopub.status.idle":"2023-10-17T00:03:48.788392Z","shell.execute_reply.started":"2023-10-17T00:03:48.772994Z","shell.execute_reply":"2023-10-17T00:03:48.787429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = de_train[['cell_type','sm_name','SMILES_vec']].copy()\ndef cell_type_merge_vec(x):\n    cell = x[0]\n    vec = x[1]\n    b_cells = 1 if cell == 'B cells' else 0\n    Myeloid = 1 if cell == 'Myeloid cells' else 0\n    other = 1 if b_cells ==0 and Myeloid == 0 else 0\n    cell_vec = np.array([b_cells,Myeloid,other])\n    merge_vec = np.concatenate([cell_vec,vec])\n    return merge_vec\n    \nid_map_smiles['input'] = id_map_smiles[['cell_type','SMILES_vec']].apply(cell_type_merge_vec,axis=1)   \nf['input'] = de_train[['cell_type','SMILES_vec']].apply(cell_type_merge_vec,axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.789545Z","iopub.execute_input":"2023-10-17T00:03:48.790481Z","iopub.status.idle":"2023-10-17T00:03:48.819268Z","shell.execute_reply.started":"2023-10-17T00:03:48.790452Z","shell.execute_reply":"2023-10-17T00:03:48.818092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_input_array = np.stack(f.input.values)\ntest_input_array = np.stack(id_map_smiles.input.values)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.820698Z","iopub.execute_input":"2023-10-17T00:03:48.821742Z","iopub.status.idle":"2023-10-17T00:03:48.830610Z","shell.execute_reply.started":"2023-10-17T00:03:48.821704Z","shell.execute_reply":"2023-10-17T00:03:48.829583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## UMAP","metadata":{}},{"cell_type":"code","source":"s_vec = np.stack(f.drop_duplicates('SMILES_vec').SMILES_vec.values)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.832072Z","iopub.execute_input":"2023-10-17T00:03:48.832356Z","iopub.status.idle":"2023-10-17T00:03:48.884633Z","shell.execute_reply.started":"2023-10-17T00:03:48.832332Z","shell.execute_reply":"2023-10-17T00:03:48.883730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s_vec.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.886312Z","iopub.execute_input":"2023-10-17T00:03:48.886760Z","iopub.status.idle":"2023-10-17T00:03:48.893191Z","shell.execute_reply.started":"2023-10-17T00:03:48.886720Z","shell.execute_reply":"2023-10-17T00:03:48.892335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reducer = umap.UMAP(random_state=126)\nem = reducer.fit_transform(s_vec)\nplt.scatter(em[:, 0], em[:, 1],c=de_train.drop_duplicates('SMILES_vec').iloc[:,6],s=5,cmap='viridis')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:48.894201Z","iopub.execute_input":"2023-10-17T00:03:48.895208Z","iopub.status.idle":"2023-10-17T00:03:57.128510Z","shell.execute_reply.started":"2023-10-17T00:03:48.895177Z","shell.execute_reply":"2023-10-17T00:03:57.127262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"em[:,0].min()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T00:03:57.133846Z","iopub.execute_input":"2023-10-17T00:03:57.134224Z","iopub.status.idle":"2023-10-17T00:03:57.142716Z","shell.execute_reply.started":"2023-10-17T00:03:57.134193Z","shell.execute_reply":"2023-10-17T00:03:57.141229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reducer = umap.UMAP(n_components=3, random_state=126)\nem = reducer.fit_transform(s_vec)\n\n\nfig = plt.figure(figsize=(12, 6)) \n\nfor i in range(1,10):\n    pos = 900 +20 + i\n    ax = fig.add_subplot(pos, projection='3d')  \n    ax.set_xlabel('UMAP Dimension 1')\n    ax.set_ylabel('UMAP Dimension 2')\n    ax.set_zlabel('UMAP Dimension 3')\n    scatter = ax.scatter(em[:, 0], em[:, 1], em[:, 2], c=de_train.drop_duplicates('SMILES_vec').iloc[:, 4+i], s=5, cmap='viridis')\n    ax.set_xlim([em[:, 0].min(), em[:, 0].max()])\n    ax.set_ylim([em[:, 1].min(), em[:, 1].max()])\n    ax.set_zlim([em[:, 2].min(), em[:, 2].max()])\n    cbar = fig.colorbar(scatter, ax=ax, orientation='vertical')\n    cbar.set_label('Value Label') \n\n    # 2つ目のサブプロットを作成。\n    ax1 = fig.add_subplot(pos+1, projection='3d')  \n    ax1.set_xlabel('UMAP Dimension 1')\n    ax1.set_ylabel('UMAP Dimension 2')\n    ax1.set_zlabel('UMAP Dimension 3')\n    ax1.scatter(em[:, 0], em[:, 1], em[:, 2], c=de_train.drop_duplicates('SMILES_vec').iloc[:, 4+i], s=5, cmap='viridis')\n    ax1.set_xlim([em[:, 0].min(), em[:, 0].max()])\n    ax1.set_ylim([em[:, 1].min(), em[:, 1].max()])\n    ax1.set_zlim([em[:, 2].min(), em[:, 2].max()])\n    ax1.view_init(elev=10, azim=90)\n\nplt.tight_layout()  \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-16T10:28:38.877339Z","iopub.execute_input":"2023-10-16T10:28:38.878580Z","iopub.status.idle":"2023-10-16T10:28:44.897283Z","shell.execute_reply.started":"2023-10-16T10:28:38.878532Z","shell.execute_reply":"2023-10-16T10:28:44.896093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reducer = umap.UMAP(n_components=3,random_state=126)\nem = reducer.fit_transform(s_vec)\nfig = plt.figure()\nax = fig.add_subplot(121, projection='3d')\nax.set_xlabel('UMAP Dimension 1')\nax.set_ylabel('UMAP Dimension 2')\nax.set_zlabel('UMAP Dimension 3')\nax.scatter(em[:, 0], em[:, 1],em[:,2],c=de_train.drop_duplicates('SMILES_vec').iloc[:,7],s=5,cmap='viridis')\nax.set_xlim([em[:,0].min(), em[:,0].max()])  # x軸の範囲を設定\nax.set_ylim([em[:,1].min(), em[:,1].max()])  # y軸の範囲を設定\nax.set_zlim([em[:,2].min(), em[:,2].max()])  # z軸の範囲を設定\nfig = plt.figure()\nax1 = fig.add_subplot(122, projection='3d')\nax1.set_xlabel('UMAP Dimension 1')\nax1.set_ylabel('UMAP Dimension 2')\nax1.set_zlabel('UMAP Dimension 3')\nax1.scatter(em[:, 0], em[:, 1],em[:,2],c=de_train.drop_duplicates('SMILES_vec').iloc[:,7],s=5,cmap='viridis')\nax1.set_xlim([em[:,0].min(), em[:,0].max()])  # x軸の範囲を設定\nax1.set_ylim([em[:,1].min(), em[:,1].max()])  # y軸の範囲を設定\nax1.set_zlim([em[:,2].min(), em[:,2].max()])  # z軸の範囲を設定\nax1.view_init(elev=10,azim=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-16T10:02:59.554175Z","iopub.execute_input":"2023-10-16T10:02:59.554582Z","iopub.status.idle":"2023-10-16T10:03:01.302824Z","shell.execute_reply.started":"2023-10-16T10:02:59.554550Z","shell.execute_reply":"2023-10-16T10:03:01.301665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reducer = umap.UMAP()\nem = reducer.fit_transform(train_input_array,y=de_train.iloc[:,5])","metadata":{"execution":{"iopub.status.busy":"2023-10-15T13:26:54.038104Z","iopub.execute_input":"2023-10-15T13:26:54.038486Z","iopub.status.idle":"2023-10-15T13:26:57.875628Z","shell.execute_reply.started":"2023-10-15T13:26:54.038459Z","shell.execute_reply":"2023-10-15T13:26:57.874489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(em[:, 0], em[:, 1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T13:26:57.877432Z","iopub.execute_input":"2023-10-15T13:26:57.877834Z","iopub.status.idle":"2023-10-15T13:26:58.104674Z","shell.execute_reply.started":"2023-10-15T13:26:57.877799Z","shell.execute_reply":"2023-10-15T13:26:58.103690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.svm import LinearSVR,SVR\nfrom sklearn.multioutput import MultiOutputRegressor\n#model = LinearSVR(max_iter=10000, epsilon=0.1)\nmodel = SVR(kernel='rbf', C=1.0, gamma='scale')\nwrapper = MultiOutputRegressor(model)\nwrapper.fit(pd.DataFrame(train_input_array),labels.iloc[:,:-1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i =0\nt = pd.DataFrame(train_input_array)\nt_feature, val_feature, t_target,val_target = train_test_split(t,labels, test_size=0.2)\nmodel = LGBM(n_estimators=1500,objective='regression',metric='RMSE',verbose=1,learning_rate=0.01)\nmodel.fit(t_feature, t_target.iloc[:,i], eval_set=[(val_feature,val_target.iloc[:,i])],\n         eval_metric='RMSE',early_stopping_rounds=50)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.barh(t_feature.columns,model.feature_importances_)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get test data \ntest_data = pd.DataFrame(id_map, columns=features_columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Auxillary functions ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\n\ndef create_model_checkpoint(filepath, monitor='val_mae', save_best_only=True,\n                            save_weights_only=True, mode='auto', verbose=0):\n    \"\"\"\n    Create a ModelCheckpoint callback for saving the best model weights during training.\n\n    Args:\n        filepath (str): Filepath to save the best weights.\n        monitor (str): Metric to monitor (e.g., 'val_loss' or 'val_mae').\n        save_best_only (bool): Save only the best weights.\n        save_weights_only (bool): Save only the model's weights, not the entire model.\n        mode (str): One of {'auto', 'min', 'max'}. In 'min' mode, it saves when the monitored metric decreases.\n        verbose (int): Verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch.\n\n    Returns:\n\n        keras.callbacks.ModelCheckpoint: ModelCheckpoint callback.\n    \"\"\"\n    checkpoint = ModelCheckpoint(\n        filepath=filepath,\n        monitor=monitor,\n        save_best_only=save_best_only,\n        save_weights_only=save_weights_only,\n        mode=mode,\n        verbose=verbose\n    )\n    return checkpoint","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_training_history(history, metrics):\n    \"\"\"\n    Plot training history curves for loss and evaluation metrics on the same line.\n\n    Args:\n        history (keras.callbacks.History): Training history object.\n        metrics (list): List of metric names to plot.\n\n    Returns:\n        None\n    \"\"\"\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n\n    epochs = range(len(loss))\n\n    plt.figure(figsize=(12, 6))\n\n    # Plot loss\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, loss, label='Training Loss', color=\"blue\")\n    plt.plot(epochs, val_loss, label='Validation Loss', color=\"red\")\n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.legend()\n\n    # Plot specified evaluation metrics on the same line\n    for metric in metrics:\n        train_metric_name = f'Training {metric.capitalize()}'\n        val_metric_name = f'Validation {metric.capitalize()}'\n        train_metric = history.history[metric]\n        val_metric = history.history['val_' + metric]\n\n        plt.subplot(1, 2, 2)\n        plt.plot(epochs, train_metric, label=train_metric_name, color=\"green\")\n        plt.plot(epochs, val_metric, label=val_metric_name, color=\"orange\")\n\n    plt.title('Metrics')\n    plt.xlabel('Epochs')\n    plt.legend(loc='upper right')\n\n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\n\ndef calculate_mae_and_mrrmse(model, data, y_true):\n    \"\"\"\n    Calculate Mean Absolute Error (MAE) and Mean Rowwise Root Mean Squared Error (MRRMSE).\n\n    Parameters:\n    - model: The trained  model.\n    - data: The input data for prediction.\n    - y_true: The true target values.\n    - scaler: The scaler used for data normalization.\n\n    Returns:\n    - None\n    \"\"\"\n    # Predict using the model\n    y_pred_original = model.predict(data, batch_size=1)\n    \n    # Calculate Mean Absolute Error (MAE)\n    mae = mean_absolute_error(y_true , y_pred_original)\n    \n    # Calculate Mean Rowwise Root Mean Squared Error (MRRMSE)\n    rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred_original), axis=1))\n    mrrmse_score = np.mean(rowwise_rmse)\n    \n    # Print the results\n    print(f\"Mean Absolute Error (MAE): {mae}\")\n    print(f\"Mean Rowwise Root Mean Squared Error (MRRMSE): {mrrmse_score}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mean_rowwise_rmse_loss(y_true, y_pred):\n    # Calculate RMSE for each row\n    rmse_per_row = tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred), axis=1))\n    # Calculate the mean of RMSE values across all rows\n    mean_rmse = tf.reduce_mean(rmse_per_row)\n    \n    return mean_rmse","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_mean_rowwise_rmse(y_true, y_pred):\n    # Calculate RMSE for each row\n    rmse_per_row = tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred), axis=1))\n    # Calculate the mean of RMSE values across all rows\n    mean_rmse = tf.reduce_mean(rmse_per_row)\n    \n    return mean_rmse","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Beautiful functions let's start experiment with building models","metadata":{}},{"cell_type":"markdown","source":"# Build the model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers import Dense, Dropout, BatchNormalization, Activation\nfrom tensorflow.keras.models import Sequential","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Model_0:** Let's start with only 2 dense layers ","metadata":{}},{"cell_type":"markdown","source":"# K-fold validation ","metadata":{}},{"cell_type":"code","source":"train_input_array.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(42)\n\nmodel = Sequential([ \n    Dense(256),\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(0.2),\n    Dense(128, activation=\"relu\"),\n    Dropout(0.2),\n    Dense(64, activation=\"relu\"),\n    BatchNormalization(),\n    Activation('relu')\n    Dropout(0.2),\n    Dense(32, activation=\"relu\"),\n    Dropout(0.2),\n    Dense(16, activation=\"relu\"),\n    Dropout(0.1),\n    Dense(18211,activation= \"linear\")\n])\n\nmodel.compile(loss=\"mae\", \n                optimizer=tf.keras.optimizers.Adam(),\n                metrics=[custom_mean_rowwise_rmse])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(42)\n\nmodel = Sequential([ \n    Dense(256, kernel_regularizer=l2(0.01)),\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(0.2),\n    \n    Dense(128, activation=\"relu\", kernel_regularizer=l2(0.01)),\n    Dropout(0.2),\n    \n    Dense(64, kernel_regularizer=l2(0.01)),\n    BatchNormalization(),\n    Activation('relu'),\n    Dropout(0.2),\n    \n    Dense(32, activation=\"relu\", kernel_regularizer=l2(0.01)),\n    Dropout(0.2),\n    \n    Dense(16, activation=\"relu\", kernel_regularizer=l2(0.01)),\n    Dropout(0.2),\n    \n    Dense(32, kernel_regularizer=l2(0.01)),\n    BatchNormalization(),\n    Activation('relu'),\n    Dropout(0.2),\n    \n    Dense(64, kernel_regularizer=l2(0.01)),\n    Activation('relu'),\n    Dropout(0.2),\n    \n    Dense(128, activation=\"relu\", kernel_regularizer=l2(0.01)),\n    Dropout(0.2),\n    \n    Dense(18211,activation= \"linear\")\n])\n\nmodel.compile(loss=\"mae\", \n                optimizer=tf.keras.optimizers.Adam(),\n                metrics=[custom_mean_rowwise_rmse])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### chemberta　version","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\n# Define the number of folds (K)\nnum_folds = 5 # You can change this value as needed\n\n# Initialize lists to store the model's performance scores\nmae_scores = []\nmrrmse_scores = []\n\n# Initialize the KFold object\nkf = KFold(n_splits=num_folds, shuffle=True, random_state=126)\n\n# Loop through the K folds\nfor train_index, val_index in kf.split(train_input_array):\n    # Convert indices to integers and split the data\n    train_index = train_index.astype(int)\n    val_index = val_index.astype(int)\n    X_train_, X_val_ = train_input_array[train_index], train_input_array[val_index]\n    y_train_, y_val_ = full_labels[train_index], full_labels[val_index]\n\n    # Train your model on X_train and y_train\n    model.fit(X_train_, y_train_, epochs=50, verbose=0)\n\n    # Make predictions on the validation set\n    y_preds = model.predict(X_val_)\n\n    # Calculate the Mean Absolute Error (MAE)\n    mae = mean_absolute_error(y_val_, y_preds)\n    mae_scores.append(mae)\n\n    # Calculate the Mean Rowwise Root Mean Square Error (MRRMSE)\n    rowwise_rmse = np.sqrt(np.mean(np.square(y_val_ - y_preds), axis=1))\n    mrrmse_score = np.mean(rowwise_rmse)\n    mrrmse_scores.append(mrrmse_score)\n\n# Calculate the mean and standard deviation of MAE and MRRMSE scores\nmean_mae = np.mean(mae_scores)\nmean_mrrmse = np.mean(mrrmse_scores)\n\n# Print the results\nprint(f'Average MAE across {num_folds} folds: {mean_mae:.4f} ')\nprint(f'Average MRRMSE across {num_folds} folds: {mean_mrrmse:.4f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train the model on full data","metadata":{}},{"cell_type":"code","source":"model.compile(loss=\"mae\", \n                optimizer=tf.keras.optimizers.Adam(),\n                metrics=[custom_mean_rowwise_rmse])\n\nhistory_4 = model.fit(train_input_array, full_labels,\n                       epochs=50,\n                       verbose=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calculate_mae_and_mrrmse(model=model, data=train_input_array, y_true=full_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicting on test data ","metadata":{}},{"cell_type":"code","source":"preds = model.predict(test_input_array, batch_size=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.shape ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_columns = sample_submission.columns\nsample_columns= sample_columns[1:]\nsubmission_df = pd.DataFrame(preds, columns=sample_columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.insert(0, 'id', range(255))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission_cb.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = list(de_train.drop('SMILES_vec',axis=1).columns[5:])\nchain = pd.read_csv('/kaggle/input/592-chain/submission-13.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if True:\n    asbl = sample_submission.copy()\n    asbl[col] = (chain[col] * 0.8) + (submission_df[col]*0.2)\n    asbl.to_csv('nn_asbl.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}