{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":67356,"databundleVersionId":8006601},{"sourceType":"datasetVersion","sourceId":8182707,"datasetId":4736625,"databundleVersionId":8305585}],"dockerImageVersionId":30618,"isInternetEnabled":true,"language":"r","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## <p style=\"border: 3px solid #3B2F2F; border-radius: 10px; padding: 15px; background-color: #99BC85; text-align: center; font-family: 'Arial', Times, serif; font-weight: bold; letter-spacing: 1px; color: #3B2F2F; font-size: 24px; margin-bottom: 10px;\"> Similarity Between Train and Test Molecules  & Generalizability of a Model</p>\n\n<h3>What's this about?</h3>\n<div style=\"line-height:24px; font-size:16px\">\n    <ul style=\"list-style:circle\">\n<li>Investigate clustering based of similarity between different molecules from train and test subsets (ideally we aim to find features capable of captiring similar patterns in train and test molecules with new BBs, since this would likely help to predict them)\n<li>Plot prediction accuracy as a function of similarity of test molecules to the nearest neighbor in the training set (the lower the similarity threshold before the accuracy rate decreases, the better the model generalizes, i.e. predicts for molecules with a new structure) - inspired by <a href=\"https://accio.github.io/AMIDD/assets/2020/13-14/McCloskey-2020-ML-DELT.pdf\">the article of McCloskey et al.</a>\n    </ul>     \n</div>","metadata":{}},{"cell_type":"code","source":"suppressMessages({    \n    if (!requireNamespace(\"BiocManager\", quietly=TRUE))\n         install.packages(\"BiocManager\")\n    BiocManager::install(\"fmcsR\", update = FALSE, quiet = TRUE) \n    library(fmcsR)\n    library(data.table)\n    library(qs)\n    library(foreach)\n    library(pheatmap)\n    library(ggplot2)\n})","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2024-07-04T09:36:24.557002Z","iopub.execute_input":"2024-07-04T09:36:24.559283Z","iopub.status.idle":"2024-07-04T09:37:56.789390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Helper functions**","metadata":{}},{"cell_type":"code","source":"shape <- function(dt) {\n    cat(\"\\nShape: \", ncol(dt), \" columns x \",\n    nrow(dt), \" rows\", sep = \"\")\n}\n\nfig <- function(width, heigth) {\n    options(repr.plot.width = width, repr.plot.height = heigth)\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:37:56.792765Z","iopub.execute_input":"2024-07-04T09:37:56.847751Z","iopub.status.idle":"2024-07-04T09:37:56.864374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****","metadata":{}},{"cell_type":"markdown","source":"# Similarity between train and test (public) molecules","metadata":{}},{"cell_type":"markdown","source":"**Load test data and subset test molecules with new BBs and shared with train set BBs**","metadata":{}},{"cell_type":"code","source":"test_dt <- fread(\"/kaggle/input/leash-BELKA/test.csv\")\nbb_dt <- fread(\"/kaggle/input/belka-supplementary-calcs-and-data-for-ml/smiles/all_bb_smiles_by_bb.csv\")\n\ntest_bbs <- bb_dt[set == \"test\", .(smiles)]\ntest_bbs <- unique(test_bbs[!bb_dt[set == \"train\", .(smiles)], on = \"smiles\"])\ntest_bbs[, .N]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:37:56.867306Z","iopub.execute_input":"2024-07-04T09:37:56.868879Z","iopub.status.idle":"2024-07-04T09:38:03.940329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mols_shared_bbs <- test_dt[! \n   (buildingblock1_smiles %chin% test_bbs$smiles |\n    buildingblock2_smiles %chin% test_bbs$smiles |\n    buildingblock3_smiles %chin% test_bbs$smiles)\n]$molecule_smiles\n\nmols_new_bbs <- test_dt[! molecule_smiles %chin% mols_shared_bbs]$molecule_smiles\n\nlength(mols_shared_bbs)\nlength(mols_new_bbs)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:38:03.944557Z","iopub.execute_input":"2024-07-04T09:38:03.946338Z","iopub.status.idle":"2024-07-04T09:38:44.535294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features <- \"ECFP\"","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:38:44.538207Z","iopub.execute_input":"2024-07-04T09:38:44.539790Z","iopub.status.idle":"2024-07-04T09:38:44.553127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load features for the test data and sample molecules for these two groups**","metadata":{}},{"cell_type":"code","source":"mols_shared_bbs <- mols_shared_bbs[sample(length(mols_shared_bbs), 500)]\nmols_new_bbs <- mols_new_bbs[sample(length(mols_new_bbs), 500)]\n\ntest_fp <- qread(\"/kaggle/input/belka-supplementary-calcs-and-data-for-ml/for_sim_analysis/mol_test_features_ecfp.qs\")\ntest_fp <- rbind(\n    test_fp[molecule_smiles %chin% mols_shared_bbs],\n    test_fp[molecule_smiles %chin% mols_new_bbs]\n    )\n\nhead(test_fp)\nshape(test_fp)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:38:44.556084Z","iopub.execute_input":"2024-07-04T09:38:44.557662Z","iopub.status.idle":"2024-07-04T09:38:59.317383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load features for the train data and sample equal number of molecules**","metadata":{}},{"cell_type":"code","source":"data <- qread(\"/kaggle/input/belka-supplementary-calcs-and-data-for-ml/for_sim_analysis/train_test_valid_280k_ecfp_1024.qs\")\ntrain_fp <- data[split == \"train\" & protein_name == \"BRD4\"][\n    sample(.N, 1000), !c(\"split\", \"buildingblock1_smiles\", \n                         \"buildingblock2_smiles\", \"buildingblock3_smiles\",\n                         \"protein_name\", \"binds\")\n]\nhead(train_fp)\nshape(train_fp)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:38:59.320280Z","iopub.execute_input":"2024-07-04T09:38:59.321808Z","iopub.status.idle":"2024-07-04T09:39:05.806808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Make a dictionary with molecule smiles and corresponding dataset names**","metadata":{}},{"cell_type":"code","source":"mols_dict <- rbind(\n    data.table(molecule_smiles = train_fp$molecule_smiles,\n              dataset = \"train\"),\n    data.table(molecule_smiles = mols_shared_bbs,\n              dataset = \"test_shared_bbs\"),\n    data.table(molecule_smiles = mols_new_bbs,\n              dataset = \"test_new_bbs\")\n    )\n\nhead(mols_dict)\nmols_dict[, uniqueN(molecule_smiles), by = \"dataset\"]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:39:05.809805Z","iopub.execute_input":"2024-07-04T09:39:05.811407Z","iopub.status.idle":"2024-07-04T09:39:05.862935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Convert train and test features to a matrix**","metadata":{}},{"cell_type":"code","source":"fp_data <- rbind(train_fp, test_fp)\nfp_mat <- as.matrix(fp_data[, -1])\nrownames(fp_mat) <- fp_data$molecule_smiles\n\nhead(fp_mat)\nshape(fp_mat)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:39:05.867235Z","iopub.execute_input":"2024-07-04T09:39:05.869155Z","iopub.status.idle":"2024-07-04T09:39:05.965876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In [the article this work was inspired by](https://accio.github.io/AMIDD/assets/2020/13-14/McCloskey-2020-ML-DELT.pdf), stated that \n\n> \"the Tanimoto similarity for two counts vectors (also commonly referred to as “1 − Jaccard Distance”) is defined as the sum of the element-wise\n> minimum of their counts divided by the sum of the element-wise maximum of their counts. A similarity value of 1.0 indicates identical structures (ignoring chirality), while 0.0 means that no substructures are shared. Nearest neighbors for hits in the training data were found using brute force exact search over the fingerprints\"\n\nSo I compare their calculations with that by fpSim","metadata":{}},{"cell_type":"code","source":"calc_tam_sim <- function(fp1, fp2) {\n  intersection <- sum(pmin(fp1, fp2))\n  union <- sum(pmax(fp1, fp2))\n  return(intersection / union)\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:39:05.970260Z","iopub.execute_input":"2024-07-04T09:39:05.972232Z","iopub.status.idle":"2024-07-04T09:39:05.988269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fpSim(fp_mat[1, ], fp_mat[15, ], method = \"Tanimoto\")\ncalc_tam_sim(fp_mat[1, ], fp_mat[15, ])","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:39:05.991532Z","iopub.execute_input":"2024-07-04T09:39:05.993111Z","iopub.status.idle":"2024-07-04T09:39:06.021677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Quite close, so I'll use fpSim","metadata":{}},{"cell_type":"markdown","source":"**Calculate Tanimoto similarity between all subsetted molecules from the train and test sets.**","metadata":{}},{"cell_type":"code","source":"tictoc::tic()\nsim_mat <- sapply(1:nrow(fp_mat), function(x) {\n    fpSim(x = fp_mat[x, ], fp_mat, sorted = FALSE, method = \"Tanimoto\")\n    })\ntictoc::toc()\n\nhead(sim_mat)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:39:06.024652Z","iopub.execute_input":"2024-07-04T09:39:06.026299Z","iopub.status.idle":"2024-07-04T09:40:57.231410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hc <- hclust(as.dist(1 - sim_mat), method = \"single\")","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:40:57.234386Z","iopub.execute_input":"2024-07-04T09:40:57.236001Z","iopub.status.idle":"2024-07-04T09:40:57.566098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig(30, 25)\n\ncolors <- colorRampPalette(c(\"white\",\"darkred\"))(10)\nheat <- pheatmap(sim_mat, fontsize = 16,\n                 color = colors,\n                 angle_col = 90,\n                 show_rownames = FALSE,\n                 show_colnames = FALSE,\n                 cluster_cols = hc,\n                 cluster_rows = FALSE,\n                 main = paste(\"Similarity using Tanimoto coefficient for\",\n                              features, \" molecular fingerprints\"))","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:40:57.569133Z","iopub.execute_input":"2024-07-04T09:40:57.570790Z","iopub.status.idle":"2024-07-04T09:41:14.580973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat(\"The number of molecules per cluster (at the level shown by the red line)\")\n\ncut_heigh = 0.57\ntable(sort(cutree(heat$tree_col, h = cut_heigh)))\n\nfig(30, 20)\nplot(heat$tree_col)\nabline(h = cut_heigh, col = \"red\", lty = 3, lwd = 1)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:41:14.584031Z","iopub.execute_input":"2024-07-04T09:41:14.585616Z","iopub.status.idle":"2024-07-04T09:41:16.633139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Make PCA and visualize clustering results.**","metadata":{}},{"cell_type":"code","source":"pca <- prcomp(sim_mat, center = TRUE, scale = TRUE)  \nplt_dt <- data.table(molecule_smiles = rownames(sim_mat),\n                     pca$x[, 1:2])\nplt_dt <- plt_dt[mols_dict, on = \"molecule_smiles\"]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:41:16.636274Z","iopub.execute_input":"2024-07-04T09:41:16.637959Z","iopub.status.idle":"2024-07-04T09:41:21.386827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ggplot(plt_dt, aes(x = PC1, y = PC2, color = dataset)) + \n    geom_point(size = 3) +\n    theme_bw(base_size = 22) +\n    theme(panel.grid = element_blank()) + #,aspect.ratio = 1\n    ggtitle(paste(\"PCA of Tanimoto coefficient for\",\n                              features, \" molecular fingerprints\")\n            )","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:41:21.390275Z","iopub.execute_input":"2024-07-04T09:41:21.392163Z","iopub.status.idle":"2024-07-04T09:41:22.806845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Ranges of similarity of test molecules with shared and new BBs to the the nearest neighbor in the training set**","metadata":{}},{"cell_type":"code","source":"diag(sim_mat) <- 0\ncolnames(sim_mat) <- rownames(sim_mat)\n\ntest_new_bbs_mat <- sim_mat[\n    rownames(sim_mat) %chin% mols_dict[dataset == \"test_new_bbs\", molecule_smiles],\n    colnames(sim_mat) %chin% mols_dict[dataset == \"train\", molecule_smiles]\n]\n\ntest_shared_bbs_mat <- sim_mat[\n    rownames(sim_mat) %chin% mols_dict[dataset == \"test_shared_bbs\", molecule_smiles],\n    colnames(sim_mat) %chin% mols_dict[dataset == \"train\", molecule_smiles]\n]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:41:50.559197Z","iopub.execute_input":"2024-07-04T09:41:50.561040Z","iopub.status.idle":"2024-07-04T09:41:50.606272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_dt_new_bbs <- data.table(\n    molecule_smiles = rownames(test_new_bbs_mat),\n    max_sim = apply(test_new_bbs_mat, 1, max),\n    median_sim = apply(test_new_bbs_mat, 1, median)\n)\nres_dt_shared_bbs <- data.table(\n    molecule_smiles = rownames(test_shared_bbs_mat),\n    max_sim = apply(test_shared_bbs_mat, 1, max),\n    median_sim = apply(test_shared_bbs_mat, 1, median)\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:43:55.648155Z","iopub.execute_input":"2024-07-04T09:43:55.650011Z","iopub.status.idle":"2024-07-04T09:43:55.801991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"head(res_dt_new_bbs)\nhead(res_dt_shared_bbs)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:44:05.804569Z","iopub.execute_input":"2024-07-04T09:44:05.806427Z","iopub.status.idle":"2024-07-04T09:44:05.853597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_dt_new_bbs[, range(max_sim)]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:44:11.663461Z","iopub.execute_input":"2024-07-04T09:44:11.665339Z","iopub.status.idle":"2024-07-04T09:44:11.686354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_dt_shared_bbs[, range(max_sim)]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:44:14.383558Z","iopub.execute_input":"2024-07-04T09:44:14.385472Z","iopub.status.idle":"2024-07-04T09:44:14.404378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_dt_new_bbs[, range(median_sim)]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:44:17.297968Z","iopub.execute_input":"2024-07-04T09:44:17.299873Z","iopub.status.idle":"2024-07-04T09:44:17.321060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_dt_shared_bbs[, range(median_sim)]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:44:23.155885Z","iopub.execute_input":"2024-07-04T09:44:23.157696Z","iopub.status.idle":"2024-07-04T09:44:23.177156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generalizability of a model","metadata":{}},{"cell_type":"markdown","source":"Here I plot prediction accuracy (measured as BCE loss) vs similarity of test molecules to the nearest neighbor in the training set. I would expect that the more similar a test molecule is to the training molecules, the more accurate the prediction would be.","metadata":{}},{"cell_type":"markdown","source":"**Load predictions and sample molecules**","metadata":{}},{"cell_type":"code","source":"preds <- qread(\"/kaggle/input/belka-supplementary-calcs-and-data-for-ml/for_sim_analysis/preds_XGb_280k_l547_lb472.qs\")\nloc_test_fp <- data[split == \"test\" & protein_name == \"BRD4\"][\n    sample(.N, 5000), !c(\"split\", \"buildingblock1_smiles\", \n                         \"buildingblock2_smiles\", \"buildingblock3_smiles\",\n                         \"protein_name\", \"binds\")\n]\npreds <- preds[loc_test_fp[, .(molecule_smiles)], on = \"molecule_smiles\"]\nhead(preds)\nshape(preds)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:45:25.003239Z","iopub.execute_input":"2024-07-04T09:45:25.005055Z","iopub.status.idle":"2024-07-04T09:45:25.266138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load equal number of train molecules**","metadata":{}},{"cell_type":"code","source":"train_fp <- data[split == \"train\" & protein_name == \"BRD4\"][\n    sample(.N, 5000), !c(\"split\", \"buildingblock1_smiles\", \n                         \"buildingblock2_smiles\", \"buildingblock3_smiles\",\n                         \"protein_name\", \"binds\")\n]\nhead(train_fp)\nshape(train_fp)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:45:25.269289Z","iopub.execute_input":"2024-07-04T09:45:25.270963Z","iopub.status.idle":"2024-07-04T09:45:26.093536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Convert train and test molecule features to matrices**","metadata":{}},{"cell_type":"code","source":"loc_test_fp_mat <- as.matrix(loc_test_fp[, -1])\nrownames(loc_test_fp_mat) <- loc_test_fp$molecule_smiles\ntrain_fp_mat <- as.matrix(train_fp[, -1])\nrownames(train_fp_mat) <- train_fp$molecule_smiles\n\nhead(loc_test_fp_mat)\nshape(loc_test_fp_mat)\nhead(train_fp_mat)\nshape(train_fp_mat)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:45:26.096748Z","iopub.execute_input":"2024-07-04T09:45:26.098455Z","iopub.status.idle":"2024-07-04T09:45:26.315899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculate Tanimoto similarity for each test molecule vs all train molecules**","metadata":{}},{"cell_type":"code","source":"tictoc::tic()\nsim_mat <- sapply(1:nrow(train_fp_mat), function(x) {\n    fpSim(x = train_fp_mat[x, ], loc_test_fp_mat, sorted = FALSE, method = \"Tanimoto\")\n    }) \ntictoc::toc()\ncolnames(sim_mat) <- rownames(train_fp_mat)\n\nhead(sim_mat)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:45:26.318940Z","iopub.execute_input":"2024-07-04T09:45:26.320712Z","iopub.status.idle":"2024-07-04T09:54:57.468161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**For each test molecule find the nearest neighbor in the training set and merge the result with predictions**","metadata":{}},{"cell_type":"code","source":"res_dt <- data.table(\n    molecule_smiles = rownames(sim_mat),\n    max_sim = apply(sim_mat, 1, max),\n    median_sim = apply(sim_mat, 1, median)\n)\nres_dt <- preds[res_dt, on = \"molecule_smiles\"]\nhead(res_dt)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:55:59.252366Z","iopub.execute_input":"2024-07-04T09:55:59.259168Z","iopub.status.idle":"2024-07-04T09:56:03.990274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculate prediction accuracy metric for each molecule**","metadata":{}},{"cell_type":"code","source":"res_dt[, bce_loss := -(as.numeric(as.character(truth)) * log(p1) + \n                       (1 - as.numeric(as.character(truth))) * log(1 - p1)\n                       )\n]\nres_dt[1:10, ]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T09:56:19.077428Z","iopub.execute_input":"2024-07-04T09:56:19.079266Z","iopub.status.idle":"2024-07-04T09:56:19.238554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Wrong predictions**","metadata":{}},{"cell_type":"code","source":"res_dt[predict != truth, .(\n    min_bce_loss = min(bce_loss),\n    max_bce_loss = max(bce_loss),\n    min_sim = min(median_sim),\n    max_sim = max(median_sim),\n    median_bce_loss = median(bce_loss),\n    median_sim = median(median_sim)\n)]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T10:01:22.445631Z","iopub.execute_input":"2024-07-04T10:01:22.448418Z","iopub.status.idle":"2024-07-04T10:01:22.492786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Correct predictions**","metadata":{}},{"cell_type":"code","source":"res_dt[predict == truth, .(\n    min_bce_loss = min(bce_loss),\n    max_bce_loss = max(bce_loss),\n    min_sim = min(median_sim),\n    max_sim = max(median_sim),\n    median_bce_loss = median(bce_loss),\n    median_sim = median(median_sim)\n)]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T10:01:56.724452Z","iopub.execute_input":"2024-07-04T10:01:56.726337Z","iopub.status.idle":"2024-07-04T10:01:56.760030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig(25, 12)\n\nggplot(res_dt, aes(x = max_sim, y = bce_loss)) +\n  geom_point() +\n  labs(x = \"Similarity to the nearest neighbor in the training set\",\n       y = \"Binary cross-entropy loss\",\n       title = \"Binary Cross-Entropy Loss vs max Similarity Score\") +\n  facet_wrap(~protein_name, scales = \"free\") +\n  theme_bw(base_size = 22)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T10:03:01.609521Z","iopub.execute_input":"2024-07-04T10:03:01.611695Z","iopub.status.idle":"2024-07-04T10:03:03.535404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig(25, 12)\n\nggplot(res_dt, aes(x = median_sim, y = bce_loss)) +\n  geom_point() +\n  labs(x = \"Similarity to the nearest neighbor in the training set\",\n       y = \"Binary cross-entropy loss\",\n       title = \"Binary Cross-Entropy Loss vs median Similarity Score\") +\n  facet_wrap(~protein_name, scales = \"free\") +\n  theme_bw(base_size = 22)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T10:03:26.005179Z","iopub.execute_input":"2024-07-04T10:03:26.007926Z","iopub.status.idle":"2024-07-04T10:03:28.237163Z"},"trusted":true},"execution_count":null,"outputs":[]}]}