{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"import polars as pl","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:28:07.828156Z","iopub.execute_input":"2024-04-08T05:28:07.828593Z","iopub.status.idle":"2024-04-08T05:28:08.116253Z","shell.execute_reply.started":"2024-04-08T05:28:07.828559Z","shell.execute_reply":"2024-04-08T05:28:08.115014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lf_train = pl.scan_parquet('/kaggle/input/leash-BELKA/train.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:28:08.501379Z","iopub.execute_input":"2024-04-08T05:28:08.501795Z","iopub.status.idle":"2024-04-08T05:28:08.579167Z","shell.execute_reply.started":"2024-04-08T05:28:08.501760Z","shell.execute_reply":"2024-04-08T05:28:08.578033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with pl.Config() as cfg:\n    cfg.set_fmt_str_lengths(100)\n    display(lf_train.head().collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:28:08.973800Z","iopub.execute_input":"2024-04-08T05:28:08.974927Z","iopub.status.idle":"2024-04-08T05:28:09.092624Z","shell.execute_reply.started":"2024-04-08T05:28:08.974883Z","shell.execute_reply":"2024-04-08T05:28:09.091749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic description","metadata":{}},{"cell_type":"markdown","source":"Null counts. Keep in mind that the dataset is huge and SMILES representations could be long strings so it would require too much memory to read all rows of SMILES columns.","metadata":{}},{"cell_type":"code","source":"display(lf_train.head(3000000).select(pl.all().null_count()).collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:31:20.980610Z","iopub.execute_input":"2024-04-08T02:31:20.981894Z","iopub.status.idle":"2024-04-08T02:31:22.726092Z","shell.execute_reply.started":"2024-04-08T02:31:20.981831Z","shell.execute_reply":"2024-04-08T02:31:22.724709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Values of `protein_name`.","metadata":{}},{"cell_type":"code","source":"display(lf_train.select(pl.col('protein_name').n_unique()).collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:31:22.728037Z","iopub.execute_input":"2024-04-08T02:31:22.728386Z","iopub.status.idle":"2024-04-08T02:31:31.018928Z","shell.execute_reply.started":"2024-04-08T02:31:22.728359Z","shell.execute_reply":"2024-04-08T02:31:31.017987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(\n    lf_train.select(pl.col('protein_name').value_counts())\n    .unnest('protein_name')\n    .sort('count', descending = True)\n    .collect()\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:31:31.020747Z","iopub.execute_input":"2024-04-08T02:31:31.021280Z","iopub.status.idle":"2024-04-08T02:31:56.027373Z","shell.execute_reply.started":"2024-04-08T02:31:31.021250Z","shell.execute_reply":"2024-04-08T02:31:56.026325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Target values.","metadata":{}},{"cell_type":"code","source":"display(\n    lf_train.select(pl.col('binds').value_counts())\n    .unnest('binds')\n    .sort('count', descending = True)\n    .collect()\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:31:56.028749Z","iopub.execute_input":"2024-04-08T02:31:56.029295Z","iopub.status.idle":"2024-04-08T02:32:08.012576Z","shell.execute_reply.started":"2024-04-08T02:31:56.029265Z","shell.execute_reply":"2024-04-08T02:32:08.011197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(pl.col('binds').mean()).collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:32:08.014841Z","iopub.execute_input":"2024-04-08T02:32:08.015464Z","iopub.status.idle":"2024-04-08T02:32:10.214938Z","shell.execute_reply.started":"2024-04-08T02:32:08.015431Z","shell.execute_reply":"2024-04-08T02:32:10.213844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.group_by('protein_name').agg(\n    pl.col('binds').mean()\n).collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:32:10.216197Z","iopub.execute_input":"2024-04-08T02:32:10.216495Z","iopub.status.idle":"2024-04-08T02:32:24.642062Z","shell.execute_reply.started":"2024-04-08T02:32:10.216467Z","shell.execute_reply":"2024-04-08T02:32:24.640639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Target is imbalanced (~0.5% positive)\n- Three targets: sEH, BRD4, and HSA.","metadata":{}},{"cell_type":"markdown","source":"Targets are arranged in a specific order: BRD4, HSA, sEH, BRD4, HSA, sEH, ...","metadata":{}},{"cell_type":"code","source":"lf_train_idx = lf_train.with_row_index('__index__')","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:32:24.645301Z","iopub.execute_input":"2024-04-08T02:32:24.646407Z","iopub.status.idle":"2024-04-08T02:32:24.651787Z","shell.execute_reply.started":"2024-04-08T02:32:24.646363Z","shell.execute_reply":"2024-04-08T02:32:24.650617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train_idx.filter(pl.col('__index__') % 3 == 0).select(\n    (pl.col('protein_name') == 'BRD4').all()\n).collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:32:55.459163Z","iopub.execute_input":"2024-04-08T02:32:55.459540Z","iopub.status.idle":"2024-04-08T02:33:01.074835Z","shell.execute_reply.started":"2024-04-08T02:32:55.459513Z","shell.execute_reply":"2024-04-08T02:33:01.073282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train_idx.filter(pl.col('__index__') % 3 == 1).select(\n    (pl.col('protein_name') == 'HSA').all()\n).collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:33:14.790662Z","iopub.execute_input":"2024-04-08T02:33:14.791132Z","iopub.status.idle":"2024-04-08T02:33:20.469826Z","shell.execute_reply.started":"2024-04-08T02:33:14.791099Z","shell.execute_reply":"2024-04-08T02:33:20.468596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train_idx.filter(pl.col('__index__') % 3 == 2).select(\n    (pl.col('protein_name') == 'sEH').all()\n).collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T02:33:27.468521Z","iopub.execute_input":"2024-04-08T02:33:27.469115Z","iopub.status.idle":"2024-04-08T02:33:32.939939Z","shell.execute_reply.started":"2024-04-08T02:33:27.469068Z","shell.execute_reply":"2024-04-08T02:33:32.938724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploration","metadata":{}},{"cell_type":"markdown","source":"Explore in the wide form.","metadata":{}},{"cell_type":"code","source":"lf_train_wide = lf_train.select(\n    pl.col('binds').gather_every(3, 0).alias('BRD4'),\n    pl.col('binds').gather_every(3, 1).alias('HSA'),\n    pl.col('binds').gather_every(3, 2).alias('sEH')\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:28:21.158074Z","iopub.execute_input":"2024-04-08T05:28:21.158673Z","iopub.status.idle":"2024-04-08T05:28:21.168912Z","shell.execute_reply.started":"2024-04-08T05:28:21.158643Z","shell.execute_reply":"2024-04-08T05:28:21.167673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train_wide.head().collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:28:21.973992Z","iopub.execute_input":"2024-04-08T05:28:21.974410Z","iopub.status.idle":"2024-04-08T05:28:30.797617Z","shell.execute_reply.started":"2024-04-08T05:28:21.974376Z","shell.execute_reply":"2024-04-08T05:28:30.796342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:28:30.800419Z","iopub.execute_input":"2024-04-08T05:28:30.801181Z","iopub.status.idle":"2024-04-08T05:28:33.322726Z","shell.execute_reply.started":"2024-04-08T05:28:30.801139Z","shell.execute_reply":"2024-04-08T05:28:33.321759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Correlation between targets.","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots(figsize = (6, 5))\nsns.heatmap(lf_train_wide.collect().to_pandas().corr(),\n            ax = ax, vmin = -1, vmax = 1, annot = True, cmap = 'coolwarm');","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:28:33.324365Z","iopub.execute_input":"2024-04-08T05:28:33.325473Z","iopub.status.idle":"2024-04-08T05:28:45.575434Z","shell.execute_reply.started":"2024-04-08T05:28:33.325433Z","shell.execute_reply":"2024-04-08T05:28:45.574297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- It seems that HSA and sEH are related.","metadata":{}},{"cell_type":"code","source":"df_target_counts = lf_train_wide.group_by('BRD4', 'HSA', 'sEH').agg(\n    pl.len().alias('count'),\n).sort('BRD4', 'HSA', 'sEH').collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:28:45.578656Z","iopub.execute_input":"2024-04-08T05:28:45.579422Z","iopub.status.idle":"2024-04-08T05:29:12.659501Z","shell.execute_reply.started":"2024-04-08T05:28:45.579384Z","shell.execute_reply":"2024-04-08T05:29:12.653107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(df_target_counts)","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:12.665122Z","iopub.execute_input":"2024-04-08T05:29:12.665588Z","iopub.status.idle":"2024-04-08T05:29:12.693050Z","shell.execute_reply.started":"2024-04-08T05:29:12.665554Z","shell.execute_reply":"2024-04-08T05:29:12.688324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Independence of targets","metadata":{}},{"cell_type":"markdown","source":"$\\chi^2$ test of independence of BRD4, HSA, and sEH. Set $\\alpha = 0.05$.","metadata":{}},{"cell_type":"code","source":"import scipy.stats","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:12.701347Z","iopub.execute_input":"2024-04-08T05:29:12.705131Z","iopub.status.idle":"2024-04-08T05:29:12.720122Z","shell.execute_reply.started":"2024-04-08T05:29:12.704829Z","shell.execute_reply":"2024-04-08T05:29:12.717552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contingency_full = df_target_counts['count'].to_numpy().reshape((2, 2, 2))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:12.724029Z","iopub.execute_input":"2024-04-08T05:29:12.725148Z","iopub.status.idle":"2024-04-08T05:29:12.744385Z","shell.execute_reply.started":"2024-04-08T05:29:12.725056Z","shell.execute_reply":"2024-04-08T05:29:12.741100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(contingency_full)","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:12.751238Z","iopub.execute_input":"2024-04-08T05:29:12.752349Z","iopub.status.idle":"2024-04-08T05:29:12.773541Z","shell.execute_reply.started":"2024-04-08T05:29:12.752301Z","shell.execute_reply":"2024-04-08T05:29:12.770155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"$H_0$: BRD4, HSA, and sEH are independent.","metadata":{}},{"cell_type":"code","source":"display(scipy.stats.chi2_contingency(contingency_full))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:12.781233Z","iopub.execute_input":"2024-04-08T05:29:12.782496Z","iopub.status.idle":"2024-04-08T05:29:12.812811Z","shell.execute_reply.started":"2024-04-08T05:29:12.782321Z","shell.execute_reply":"2024-04-08T05:29:12.809627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We reject the null hypothesis that BRD4, HSA, and sEH are independent.","metadata":{}},{"cell_type":"markdown","source":"$H_0$: HSA and sEH are independent.","metadata":{}},{"cell_type":"code","source":"display(scipy.stats.chi2_contingency(contingency_full.sum(axis = 0)))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:12.823540Z","iopub.execute_input":"2024-04-08T05:29:12.824955Z","iopub.status.idle":"2024-04-08T05:29:12.851370Z","shell.execute_reply.started":"2024-04-08T05:29:12.824799Z","shell.execute_reply":"2024-04-08T05:29:12.847074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(scipy.stats.contingency.association(contingency_full.sum(axis = 0)))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:49.342033Z","iopub.execute_input":"2024-04-08T05:29:49.342486Z","iopub.status.idle":"2024-04-08T05:29:49.351412Z","shell.execute_reply.started":"2024-04-08T05:29:49.342450Z","shell.execute_reply":"2024-04-08T05:29:49.350178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We reject the null hypothesis that HSA and sEH are independent.","metadata":{}},{"cell_type":"markdown","source":"$H_0$: BRD4 and sEH are independent","metadata":{}},{"cell_type":"code","source":"display(scipy.stats.chi2_contingency(contingency_full.sum(axis = 1)))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:12.854799Z","iopub.execute_input":"2024-04-08T05:29:12.855415Z","iopub.status.idle":"2024-04-08T05:29:12.876887Z","shell.execute_reply.started":"2024-04-08T05:29:12.855385Z","shell.execute_reply":"2024-04-08T05:29:12.874320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(scipy.stats.contingency.association(contingency_full.sum(axis = 1)))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:31:43.559189Z","iopub.execute_input":"2024-04-08T05:31:43.559641Z","iopub.status.idle":"2024-04-08T05:31:43.568261Z","shell.execute_reply.started":"2024-04-08T05:31:43.559611Z","shell.execute_reply":"2024-04-08T05:31:43.567222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We reject the null hypothesis that BRD4 and sEH are independent.","metadata":{}},{"cell_type":"markdown","source":"$H_0$: BRD4 and HSA are independent.","metadata":{}},{"cell_type":"code","source":"display(scipy.stats.chi2_contingency(contingency_full.sum(axis = 2)))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:29:12.881314Z","iopub.execute_input":"2024-04-08T05:29:12.882477Z","iopub.status.idle":"2024-04-08T05:29:12.905941Z","shell.execute_reply.started":"2024-04-08T05:29:12.882394Z","shell.execute_reply":"2024-04-08T05:29:12.903281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(scipy.stats.contingency.association(contingency_full.sum(axis = 2)))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T05:32:14.375888Z","iopub.execute_input":"2024-04-08T05:32:14.376345Z","iopub.status.idle":"2024-04-08T05:32:14.384678Z","shell.execute_reply.started":"2024-04-08T05:32:14.376310Z","shell.execute_reply":"2024-04-08T05:32:14.383351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We reject the null hypothesis that BRD4 and HSA are independent.","metadata":{}},{"cell_type":"markdown","source":"Summary:\n- We reject independence of each of (HSA, sEH), (BRD4, sEH), (BRD4, HSA) pairs.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}