{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":6538067,"sourceType":"datasetVersion","datasetId":3779782},{"sourceId":6583269,"sourceType":"datasetVersion","datasetId":3800631},{"sourceId":6617622,"sourceType":"datasetVersion","datasetId":3819227},{"sourceId":6801987,"sourceType":"datasetVersion","datasetId":3913683}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings \nwarnings.filterwarnings('ignore')\n\nimport os\nimport gc\nimport glob\nimport random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom scipy import stats\nfrom pathlib import Path\nfrom itertools import groupby\n\nimport matplotlib.pyplot as plt\nimport plotly.figure_factory as ff\nimport plotly.express as px\n\n%matplotlib inline\n\nimport matplotlib.style as style\nstyle.use('dark_background')\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"papermill":{"duration":5.485488,"end_time":"2023-11-04T08:59:15.108698","exception":false,"start_time":"2023-11-04T08:59:09.623210","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:19.184467Z","iopub.execute_input":"2023-11-16T09:04:19.185358Z","iopub.status.idle":"2023-11-16T09:04:19.213826Z","shell.execute_reply.started":"2023-11-16T09:04:19.185318Z","shell.execute_reply":"2023-11-16T09:04:19.212988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train = pd.read_parquet('../input/open-problems-single-cell-perturbations/de_train.parquet')\nid_map = pd.read_csv('../input/open-problems-single-cell-perturbations/id_map.csv')\nsample_submission = pd.read_csv('../input/open-problems-single-cell-perturbations/sample_submission.csv', index_col='id')","metadata":{"papermill":{"duration":2.623274,"end_time":"2023-11-04T08:59:17.779410","exception":false,"start_time":"2023-11-04T08:59:15.156136","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:19.367336Z","iopub.execute_input":"2023-11-16T09:04:19.368039Z","iopub.status.idle":"2023-11-16T09:04:24.575836Z","shell.execute_reply.started":"2023-11-16T09:04:19.368002Z","shell.execute_reply":"2023-11-16T09:04:24.574995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"de_train - {de_train.shape}\")\nprint(f\"id_map - {id_map.shape}\")\nprint(f\"sample_submission - {sample_submission.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:04:24.577362Z","iopub.execute_input":"2023-11-16T09:04:24.577664Z","iopub.status.idle":"2023-11-16T09:04:24.582864Z","shell.execute_reply.started":"2023-11-16T09:04:24.577638Z","shell.execute_reply":"2023-11-16T09:04:24.581657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xlist = ['cell_type', 'sm_name']\n_ylist = ['cell_type', 'sm_name', 'sm_lincs_id', 'SMILES', 'control']\n\ny = de_train.drop(columns=_ylist)\n\ny.shape","metadata":{"papermill":{"duration":0.075475,"end_time":"2023-11-04T08:59:22.914092","exception":false,"start_time":"2023-11-04T08:59:22.838617","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.584315Z","iopub.execute_input":"2023-11-16T09:04:24.584731Z","iopub.status.idle":"2023-11-16T09:04:24.642895Z","shell.execute_reply.started":"2023-11-16T09:04:24.584704Z","shell.execute_reply":"2023-11-16T09:04:24.641917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.get_dummies(de_train[xlist], columns=xlist)\n\ntrain.shape","metadata":{"papermill":{"duration":0.035001,"end_time":"2023-11-04T08:59:23.063127","exception":false,"start_time":"2023-11-04T08:59:23.028126","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.645692Z","iopub.execute_input":"2023-11-16T09:04:24.646348Z","iopub.status.idle":"2023-11-16T09:04:24.658362Z","shell.execute_reply.started":"2023-11-16T09:04:24.646320Z","shell.execute_reply":"2023-11-16T09:04:24.657501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.get_dummies(id_map[xlist], columns=xlist)\n\ntest.shape","metadata":{"papermill":{"duration":0.031909,"end_time":"2023-11-04T08:59:23.146080","exception":false,"start_time":"2023-11-04T08:59:23.114171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.659671Z","iopub.execute_input":"2023-11-16T09:04:24.660081Z","iopub.status.idle":"2023-11-16T09:04:24.674916Z","shell.execute_reply.started":"2023-11-16T09:04:24.660050Z","shell.execute_reply":"2023-11-16T09:04:24.673984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uncommon = [f for f in train if f not in test]\n\nlen(uncommon)","metadata":{"papermill":{"duration":0.028677,"end_time":"2023-11-04T08:59:23.226010","exception":false,"start_time":"2023-11-04T08:59:23.197333","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.676101Z","iopub.execute_input":"2023-11-16T09:04:24.676429Z","iopub.status.idle":"2023-11-16T09:04:24.683544Z","shell.execute_reply.started":"2023-11-16T09:04:24.676404Z","shell.execute_reply":"2023-11-16T09:04:24.682603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop(columns=uncommon)\n\nX.shape[1], test.shape[1]","metadata":{"papermill":{"duration":0.031337,"end_time":"2023-11-04T08:59:23.308705","exception":false,"start_time":"2023-11-04T08:59:23.277368","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.684846Z","iopub.execute_input":"2023-11-16T09:04:24.685125Z","iopub.status.idle":"2023-11-16T09:04:24.698635Z","shell.execute_reply.started":"2023-11-16T09:04:24.685101Z","shell.execute_reply":"2023-11-16T09:04:24.697750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(X.columns) == list(test.columns)","metadata":{"papermill":{"duration":0.027774,"end_time":"2023-11-04T08:59:23.388358","exception":false,"start_time":"2023-11-04T08:59:23.360584","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.699850Z","iopub.execute_input":"2023-11-16T09:04:24.700440Z","iopub.status.idle":"2023-11-16T09:04:24.709188Z","shell.execute_reply.started":"2023-11-16T09:04:24.700408Z","shell.execute_reply":"2023-11-16T09:04:24.708332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mrrmse_pd(y_pred: pd.DataFrame, y_true: pd.DataFrame):\n    return ((y_pred - y_true)**2).mean(axis=1).apply(np.sqrt).mean()","metadata":{"papermill":{"duration":0.028794,"end_time":"2023-11-04T08:59:23.469750","exception":false,"start_time":"2023-11-04T08:59:23.440956","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.710496Z","iopub.execute_input":"2023-11-16T09:04:24.710836Z","iopub.status.idle":"2023-11-16T09:04:24.719879Z","shell.execute_reply.started":"2023-11-16T09:04:24.710806Z","shell.execute_reply":"2023-11-16T09:04:24.719063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mrrmse_np(y_pred, y_true):\n    return np.sqrt(np.square(y_true - y_pred).mean(axis=1)).mean()","metadata":{"papermill":{"duration":0.028437,"end_time":"2023-11-04T08:59:23.551305","exception":false,"start_time":"2023-11-04T08:59:23.522868","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.723743Z","iopub.execute_input":"2023-11-16T09:04:24.724316Z","iopub.status.idle":"2023-11-16T09:04:24.730065Z","shell.execute_reply.started":"2023-11-16T09:04:24.724291Z","shell.execute_reply":"2023-11-16T09:04:24.729365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_cell_type = de_train.iloc[:, [0] + list(range(5, de_train.shape[1]))]\n\nde_sm_name = de_train.iloc[:, [1] + list(range(5, de_train.shape[1]))]\n\nde_cell_type.shape, de_sm_name.shape\n","metadata":{"papermill":{"duration":0.123346,"end_time":"2023-11-04T08:59:23.729676","exception":false,"start_time":"2023-11-04T08:59:23.606330","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.730976Z","iopub.execute_input":"2023-11-16T09:04:24.731209Z","iopub.status.idle":"2023-11-16T09:04:24.814478Z","shell.execute_reply.started":"2023-11-16T09:04:24.731188Z","shell.execute_reply":"2023-11-16T09:04:24.813570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_cell_type = de_cell_type.groupby('cell_type').mean().reset_index()\nmean_sm_name = de_sm_name.groupby('sm_name').mean().reset_index()\n\ndisplay(mean_cell_type)\n\ndisplay(mean_sm_name)","metadata":{"papermill":{"duration":0.44656,"end_time":"2023-11-04T08:59:24.231415","exception":false,"start_time":"2023-11-04T08:59:23.784855","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:24.815686Z","iopub.execute_input":"2023-11-16T09:04:24.815998Z","iopub.status.idle":"2023-11-16T09:04:25.170069Z","shell.execute_reply.started":"2023-11-16T09:04:24.815967Z","shell.execute_reply":"2023-11-16T09:04:25.169161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = []\nfor name in de_cell_type['cell_type']:\n    mean_rows = mean_cell_type[mean_cell_type['cell_type'] == name].copy()\n    rows.append(mean_rows)","metadata":{"papermill":{"duration":0.595897,"end_time":"2023-11-04T08:59:24.885152","exception":false,"start_time":"2023-11-04T08:59:24.289255","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:25.171435Z","iopub.execute_input":"2023-11-16T09:04:25.171845Z","iopub.status.idle":"2023-11-16T09:04:25.634301Z","shell.execute_reply.started":"2023-11-16T09:04:25.171807Z","shell.execute_reply":"2023-11-16T09:04:25.633456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_cell_type = pd.concat(rows)\ntr_cell_type = tr_cell_type.reset_index(drop=True)\n\ntr_cell_type","metadata":{"papermill":{"duration":1.16046,"end_time":"2023-11-04T08:59:26.103659","exception":false,"start_time":"2023-11-04T08:59:24.943199","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:25.635330Z","iopub.execute_input":"2023-11-16T09:04:25.635621Z","iopub.status.idle":"2023-11-16T09:04:26.621134Z","shell.execute_reply.started":"2023-11-16T09:04:25.635596Z","shell.execute_reply":"2023-11-16T09:04:26.620179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = []\nfor name in de_sm_name['sm_name']:\n    mean_rows = mean_sm_name[mean_sm_name['sm_name'] == name].copy()\n    rows.append(mean_rows)","metadata":{"papermill":{"duration":0.671499,"end_time":"2023-11-04T08:59:26.836001","exception":false,"start_time":"2023-11-04T08:59:26.164502","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:26.622503Z","iopub.execute_input":"2023-11-16T09:04:26.622869Z","iopub.status.idle":"2023-11-16T09:04:27.066772Z","shell.execute_reply.started":"2023-11-16T09:04:26.622835Z","shell.execute_reply":"2023-11-16T09:04:27.065789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_sm_name = pd.concat(rows)\ntr_sm_name = tr_sm_name.reset_index(drop=True)\n\ntr_sm_name","metadata":{"papermill":{"duration":1.27575,"end_time":"2023-11-04T08:59:28.171949","exception":false,"start_time":"2023-11-04T08:59:26.896199","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:27.068269Z","iopub.execute_input":"2023-11-16T09:04:27.068604Z","iopub.status.idle":"2023-11-16T09:04:28.059905Z","shell.execute_reply.started":"2023-11-16T09:04:27.068577Z","shell.execute_reply":"2023-11-16T09:04:28.058932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = []\nfor name in id_map['cell_type']:\n    mean_rows = mean_cell_type[mean_cell_type['cell_type'] == name].copy()\n    rows.append(mean_rows)","metadata":{"papermill":{"duration":0.233656,"end_time":"2023-11-04T08:59:28.469024","exception":false,"start_time":"2023-11-04T08:59:28.235368","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:28.061282Z","iopub.execute_input":"2023-11-16T09:04:28.061693Z","iopub.status.idle":"2023-11-16T09:04:28.220032Z","shell.execute_reply.started":"2023-11-16T09:04:28.061653Z","shell.execute_reply":"2023-11-16T09:04:28.219007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"te_cell_type = pd.concat(rows)\nte_cell_type = te_cell_type.reset_index(drop=True)\n\nte_cell_type","metadata":{"papermill":{"duration":0.514418,"end_time":"2023-11-04T08:59:29.046088","exception":false,"start_time":"2023-11-04T08:59:28.531670","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:28.221203Z","iopub.execute_input":"2023-11-16T09:04:28.221536Z","iopub.status.idle":"2023-11-16T09:04:28.633699Z","shell.execute_reply.started":"2023-11-16T09:04:28.221511Z","shell.execute_reply":"2023-11-16T09:04:28.632749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = []\nfor name in id_map['sm_name']:\n    mean_rows = mean_sm_name[mean_sm_name['sm_name'] == name].copy()\n    rows.append(mean_rows)\n\nte_sm_name = pd.concat(rows)\nte_sm_name = te_sm_name.reset_index(drop=True)\n\nte_sm_name","metadata":{"papermill":{"duration":0.726255,"end_time":"2023-11-04T08:59:29.837445","exception":false,"start_time":"2023-11-04T08:59:29.111190","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:56.375975Z","iopub.execute_input":"2023-11-16T09:04:56.376365Z","iopub.status.idle":"2023-11-16T09:04:56.970759Z","shell.execute_reply.started":"2023-11-16T09:04:56.376332Z","shell.execute_reply":"2023-11-16T09:04:56.969763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y0 = y.iloc[:, 0].copy()\n\ny0","metadata":{"papermill":{"duration":0.037836,"end_time":"2023-11-04T08:59:29.942621","exception":false,"start_time":"2023-11-04T08:59:29.904785","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:58.540152Z","iopub.execute_input":"2023-11-16T09:04:58.540771Z","iopub.status.idle":"2023-11-16T09:04:58.550110Z","shell.execute_reply.started":"2023-11-16T09:04:58.540735Z","shell.execute_reply":"2023-11-16T09:04:58.548969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X0 = X.join(tr_cell_type.iloc[:, 0+1]).copy()\nX0 = X0.join(tr_sm_name.iloc[:, 0+1], lsuffix='_cell_type', rsuffix='_sm_name')\n\nX0","metadata":{"papermill":{"duration":0.069225,"end_time":"2023-11-04T08:59:30.081234","exception":false,"start_time":"2023-11-04T08:59:30.012009","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:04:59.655969Z","iopub.execute_input":"2023-11-16T09:04:59.656664Z","iopub.status.idle":"2023-11-16T09:04:59.690619Z","shell.execute_reply.started":"2023-11-16T09:04:59.656632Z","shell.execute_reply":"2023-11-16T09:04:59.689668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X0","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:05:06.586808Z","iopub.execute_input":"2023-11-16T09:05:06.587152Z","iopub.status.idle":"2023-11-16T09:05:06.616279Z","shell.execute_reply.started":"2023-11-16T09:05:06.587126Z","shell.execute_reply":"2023-11-16T09:05:06.615264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test0 = test.join(te_cell_type.iloc[:, 0+1]).copy()\ntest0 = test0.join(te_sm_name.iloc[:, 0+1], lsuffix='_cell_type', rsuffix='_sm_name')\n\ntest0","metadata":{"papermill":{"duration":0.068656,"end_time":"2023-11-04T08:59:30.221093","exception":false,"start_time":"2023-11-04T08:59:30.152437","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:05:07.519509Z","iopub.execute_input":"2023-11-16T09:05:07.519871Z","iopub.status.idle":"2023-11-16T09:05:07.555911Z","shell.execute_reply.started":"2023-11-16T09:05:07.519842Z","shell.execute_reply":"2023-11-16T09:05:07.554902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X0_corr = X0.copy()\nX0_corr['y0'] = y0\n\ncorr = X0_corr.iloc[:, 131:].corr(numeric_only=True).round(3)\n\ncorr","metadata":{"papermill":{"duration":0.102345,"end_time":"2023-11-04T08:59:30.397010","exception":false,"start_time":"2023-11-04T08:59:30.294665","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:05:08.698582Z","iopub.execute_input":"2023-11-16T09:05:08.699394Z","iopub.status.idle":"2023-11-16T09:05:08.712530Z","shell.execute_reply.started":"2023-11-16T09:05:08.699360Z","shell.execute_reply":"2023-11-16T09:05:08.711449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cor_matrix = X0_corr.iloc[:, 131:].corr()\n\nfig = plt.figure(figsize=(6, 6))\ncmap = sns.diverging_palette(240, 10, s=75, l=50, sep=1, n=6, center='light', as_cmap=False)\nsns.heatmap(cor_matrix, center=0, annot=True, cmap=cmap, linewidths=5)\n\nplt.suptitle('Train Set (Heatmap)', y=0.92, fontsize=16)\nplt.show()\n","metadata":{"papermill":{"duration":0.416585,"end_time":"2023-11-04T08:59:30.888214","exception":false,"start_time":"2023-11-04T08:59:30.471629","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:05:08.866129Z","iopub.execute_input":"2023-11-16T09:05:08.866433Z","iopub.status.idle":"2023-11-16T09:05:09.188134Z","shell.execute_reply.started":"2023-11-16T09:05:08.866407Z","shell.execute_reply":"2023-11-16T09:05:09.187280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Machine Learning","metadata":{"papermill":{"duration":0.025236,"end_time":"2023-11-04T08:59:30.939987","exception":false,"start_time":"2023-11-04T08:59:30.914751","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X0, y0, test_size=0.20, random_state=421)","metadata":{"papermill":{"duration":1.656271,"end_time":"2023-11-04T08:59:32.621695","exception":false,"start_time":"2023-11-04T08:59:30.965424","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:05:09.192182Z","iopub.execute_input":"2023-11-16T09:05:09.192443Z","iopub.status.idle":"2023-11-16T09:05:09.199122Z","shell.execute_reply.started":"2023-11-16T09:05:09.192420Z","shell.execute_reply":"2023-11-16T09:05:09.198173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.preprocessing import MinMaxScaler\n\nmodel = Pipeline([\n    ('scaler', MinMaxScaler()),\n    ('regressor', DecisionTreeRegressor(criterion='friedman_mse', max_features=None, min_samples_leaf=0.029530863384643957, min_samples_split=0.000630957344480193, splitter='best'))\n])\n\nmodel.fit(X_train, y_train)","metadata":{"papermill":{"duration":0.431007,"end_time":"2023-11-04T08:59:33.080724","exception":false,"start_time":"2023-11-04T08:59:32.649717","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:05:09.370813Z","iopub.execute_input":"2023-11-16T09:05:09.371105Z","iopub.status.idle":"2023-11-16T09:05:09.396883Z","shell.execute_reply.started":"2023-11-16T09:05:09.371080Z","shell.execute_reply":"2023-11-16T09:05:09.395871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(X_test)","metadata":{"papermill":{"duration":0.038474,"end_time":"2023-11-04T08:59:33.145100","exception":false,"start_time":"2023-11-04T08:59:33.106626","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:05:14.932300Z","iopub.execute_input":"2023-11-16T09:05:14.932674Z","iopub.status.idle":"2023-11-16T09:05:14.941294Z","shell.execute_reply.started":"2023-11-16T09:05:14.932641Z","shell.execute_reply":"2023-11-16T09:05:14.940134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = 0\n\nplt.figure(figsize=(8, 4))\n\nplt.gca()\n\nsns.distplot(y_test.values - predictions, bins=100, color='red')\n\nplt.legend(['y_true', 'y_pred'], loc=1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:07:33.561272Z","iopub.execute_input":"2023-11-16T09:07:33.561655Z","iopub.status.idle":"2023-11-16T09:07:33.996193Z","shell.execute_reply.started":"2023-11-16T09:07:33.561623Z","shell.execute_reply":"2023-11-16T09:07:33.995198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final","metadata":{"papermill":{"duration":0.026207,"end_time":"2023-11-04T08:59:33.830577","exception":false,"start_time":"2023-11-04T08:59:33.804370","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model = Pipeline([\n    ('scaler', MinMaxScaler()),\n    ('regressor', DecisionTreeRegressor(criterion='friedman_mse', max_features=None, min_samples_leaf=0.029530863384643957, min_samples_split=0.000630957344480193, splitter='best'))\n])","metadata":{"papermill":{"duration":0.037756,"end_time":"2023-11-04T08:59:33.894691","exception":false,"start_time":"2023-11-04T08:59:33.856935","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:07:53.806660Z","iopub.execute_input":"2023-11-16T09:07:53.807522Z","iopub.status.idle":"2023-11-16T09:07:53.812277Z","shell.execute_reply.started":"2023-11-16T09:07:53.807480Z","shell.execute_reply":"2023-11-16T09:07:53.811289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\npred = []\n\nfor i in tqdm(range(y.shape[1])):\n    \n    yi = y.iloc[:, i].copy()\n    \n    Xi = X.join(tr_cell_type.iloc[:, i+1]).copy()\n    Xi = Xi.join(tr_sm_name.iloc[:, i+1], lsuffix='_cell_type', rsuffix='_sm_name')\n    \n    testi = test.join(te_cell_type.iloc[:, i+1]).copy()\n    testi = testi.join(te_sm_name.iloc[:, i+1], lsuffix='_cell_type', rsuffix='_sm_name')\n    \n    model.fit(Xi, yi)\n    pred.append(model.predict(testi))\n    \nprint(f\"Length of predictions: {len(pred)}\")\n","metadata":{"papermill":{"duration":8101.62114,"end_time":"2023-11-04T11:14:35.542273","exception":false,"start_time":"2023-11-04T08:59:33.921133","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:08:52.725741Z","iopub.execute_input":"2023-11-16T09:08:52.726414Z","iopub.status.idle":"2023-11-16T09:08:59.278519Z","shell.execute_reply.started":"2023-11-16T09:08:52.726380Z","shell.execute_reply":"2023-11-16T09:08:59.276969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = pd.DataFrame(pred).T\n\nprediction.columns = de_train.columns[5:]\nprediction.index.name = 'id'\n\nprediction","metadata":{"papermill":{"duration":3.13313,"end_time":"2023-11-04T11:14:38.701888","exception":false,"start_time":"2023-11-04T11:14:35.568758","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T08:57:09.377928Z","iopub.execute_input":"2023-11-16T08:57:09.378800Z","iopub.status.idle":"2023-11-16T08:57:11.600591Z","shell.execute_reply.started":"2023-11-16T08:57:09.378765Z","shell.execute_reply":"2023-11-16T08:57:11.599490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction.to_csv('prediction.csv')\n\n!ls","metadata":{"papermill":{"duration":14.419795,"end_time":"2023-11-04T11:14:53.149080","exception":false,"start_time":"2023-11-04T11:14:38.729285","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T08:57:21.123655Z","iopub.execute_input":"2023-11-16T08:57:21.124283Z","iopub.status.idle":"2023-11-16T08:57:33.807322Z","shell.execute_reply.started":"2023-11-16T08:57:21.124248Z","shell.execute_reply":"2023-11-16T08:57:33.805690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing Data and Combining Submissions 📓","metadata":{"papermill":{"duration":0.027621,"end_time":"2023-11-04T11:14:53.204737","exception":false,"start_time":"2023-11-04T11:14:53.177116","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import1 = pd.read_csv('../input/op2-603/op2_603.csv', index_col='id')\nimport1.shape\n\nimport2 = pd.read_csv('../input/op2-720/op2_720.csv', index_col='id')\nimport2.shape\n\nimport3 = pd.read_csv('../input/op2-604/submission_df.csv', index_col='id')\nimport3.shape\n\nimport4 = pd.read_csv('../input/op2-607/OP2_607.csv', index_col='id')\nimport4.shape\n\ncol = list(de_train.columns[5:])\n\nsubmission = sample_submission.copy()\n\nsubmission[col] = (import1[col] * 0.1) + (import2[col] * 0.1) + (import3[col] * 0.1) + (import4[col] * 0.1) + (prediction[col] * 0.6)\n\nsubmission.shape","metadata":{"papermill":{"duration":35.239814,"end_time":"2023-11-04T11:15:28.472240","exception":false,"start_time":"2023-11-04T11:14:53.232426","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T08:59:35.754105Z","iopub.execute_input":"2023-11-16T08:59:35.755022Z","iopub.status.idle":"2023-11-16T09:00:06.420374Z","shell.execute_reply.started":"2023-11-16T08:59:35.754976Z","shell.execute_reply":"2023-11-16T09:00:06.419422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv')\n\n!ls","metadata":{"papermill":{"duration":52.400136,"end_time":"2023-11-04T11:16:20.900199","exception":false,"start_time":"2023-11-04T11:15:28.500063","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T09:00:18.636381Z","iopub.execute_input":"2023-11-16T09:00:18.637432Z","iopub.status.idle":"2023-11-16T09:01:03.429031Z","shell.execute_reply.started":"2023-11-16T09:00:18.637394Z","shell.execute_reply":"2023-11-16T09:01:03.427992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}