{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport glob\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:10:55.525218Z","iopub.execute_input":"2022-11-02T11:10:55.525818Z","iopub.status.idle":"2022-11-02T11:10:55.559881Z","shell.execute_reply.started":"2022-11-02T11:10:55.525743Z","shell.execute_reply":"2022-11-02T11:10:55.558988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install/import","metadata":{}},{"cell_type":"code","source":"\nimport time\nt0start = time.time()\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport sys\n\nimport matplotlib.pyplot as plt\n#plt.style.use('dark_background')\nimport seaborn as sns\n\n#If you see a urllib warning running this cell, go to \"Settings\" on the right hand side, \n#and turn on internet. Note, you need to be phone verified.\n!pip install --quiet tables\n\n\nimport h5py\n!pip install hdf5plugin~=2.0 # https://forum.hdfgroup.org/t/cant-open-directory-usr-local-hdf5-lib-plugin/9738/4\nimport hdf5plugin\n\n# !pip install scanpy\n# import scanpy as sc\n# import anndata\n\nDATA_DIR = \"/kaggle/input/open-problems-multimodal/\"","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:10:55.561509Z","iopub.execute_input":"2022-11-02T11:10:55.562998Z","iopub.status.idle":"2022-11-02T11:11:17.447685Z","shell.execute_reply.started":"2022-11-02T11:10:55.562934Z","shell.execute_reply":"2022-11-02T11:11:17.446407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:11:17.449345Z","iopub.execute_input":"2022-11-02T11:11:17.449681Z","iopub.status.idle":"2022-11-02T11:11:17.458121Z","shell.execute_reply.started":"2022-11-02T11:11:17.449654Z","shell.execute_reply":"2022-11-02T11:11:17.456259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data\n","metadata":{}},{"cell_type":"code","source":"y = pd.read_csv('/kaggle/input/tuning-dataset/enriched_df_y.csv', index_col=0).values\nX = pd.read_csv('/kaggle/input/tuning-dataset/enriched_df.csv', index_col=0).values","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:11:17.461282Z","iopub.execute_input":"2022-11-02T11:11:17.461613Z","iopub.status.idle":"2022-11-02T11:12:29.116279Z","shell.execute_reply.started":"2022-11-02T11:11:17.461579Z","shell.execute_reply":"2022-11-02T11:12:29.11557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\n# /kaggle/input/tuning-dataset/list_folds_indices_by_days_and_donors.pkl\n# /kaggle/input/tuning-dataset/list_folds_indices_by_days_and_donors_with2holdouts.pkl\n\nwith open('../input/tuning-dataset/list_folds_indices_by_days_and_donors.pkl', 'rb') as f:\n    list_folds_indices_by_days_and_donors = pickle.load(f)\n    \nwith open('../input/tuning-dataset/list_folds_indices_by_days_and_donors_with2holdouts.pkl', 'rb') as f:\n    list_folds_indices_by_days_and_donors_with2holdouts = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:29.117699Z","iopub.execute_input":"2022-11-02T11:12:29.118337Z","iopub.status.idle":"2022-11-02T11:12:29.184702Z","shell.execute_reply.started":"2022-11-02T11:12:29.118302Z","shell.execute_reply":"2022-11-02T11:12:29.183001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optional Rescaling of Targets","metadata":{}},{"cell_type":"markdown","source":"# Features by TruncatedSVD, modeling - Ridge. \n\n\nConclusion - model peforms better on \"priviate like\" test (unseen day&donor), rather than on just unseen donor - quite strange. \nBut it is only for \"r2\"-metric, not for correlation metric \n","metadata":{}},{"cell_type":"markdown","source":"# Concatenating additional features ","metadata":{}},{"cell_type":"code","source":"!pip install pytorch-tabnet","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:29.186051Z","iopub.execute_input":"2022-11-02T11:12:29.186304Z","iopub.status.idle":"2022-11-02T11:12:38.489909Z","shell.execute_reply.started":"2022-11-02T11:12:29.186281Z","shell.execute_reply":"2022-11-02T11:12:38.489198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\n\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:38.490964Z","iopub.execute_input":"2022-11-02T11:12:38.491297Z","iopub.status.idle":"2022-11-02T11:12:40.274242Z","shell.execute_reply.started":"2022-11-02T11:12:38.49125Z","shell.execute_reply":"2022-11-02T11:12:40.273222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = r\n# first_batch = list_folds_indices_by_days_and_donors\nindices_tuple = next(iter(list_folds_indices_by_days_and_donors))\nmain_test_index = indices_tuple[1]\ntrain_index = indices_tuple[0]\nt_index, v_index = train_test_split(train_index)\nX_train = X[t_index]\ny_train = y[t_index]\nX_valid = X[v_index]\ny_valid = y[v_index]","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:40.275775Z","iopub.execute_input":"2022-11-02T11:12:40.27644Z","iopub.status.idle":"2022-11-02T11:12:40.919378Z","shell.execute_reply.started":"2022-11-02T11:12:40.276414Z","shell.execute_reply":"2022-11-02T11:12:40.918153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(X_train[np.isnan(X_train)]) == 0\nassert len(y_train[np.isnan(y_train)]) == 0\n\nassert len(X_valid[np.isnan(X_valid)]) == 0\nassert len(y_valid[np.isnan(y_valid)]) == 0","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:40.920503Z","iopub.execute_input":"2022-11-02T11:12:40.920736Z","iopub.status.idle":"2022-11-02T11:12:40.990701Z","shell.execute_reply.started":"2022-11-02T11:12:40.920713Z","shell.execute_reply":"2022-11-02T11:12:40.989688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:40.993233Z","iopub.execute_input":"2022-11-02T11:12:40.993469Z","iopub.status.idle":"2022-11-02T11:12:41.415418Z","shell.execute_reply.started":"2022-11-02T11:12:40.993446Z","shell.execute_reply":"2022-11-02T11:12:41.414109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim import Optimizer\n\ndef exists(val):\n    return val is not None\n\nclass Adan(Optimizer):\n    def __init__(\n        self,\n        params,\n        lr = 1e-3,\n        betas = (0.02, 0.08, 0.01),\n        eps = 1e-8,\n        weight_decay = 0,\n        restart_cond: callable = None\n    ):\n        assert len(betas) == 3\n\n        defaults = dict(\n            lr = lr,\n            betas = betas,\n            eps = eps,\n            weight_decay = weight_decay,\n            restart_cond = restart_cond\n        )\n\n        super().__init__(params, defaults)\n\n    def step(self, closure = None):\n        loss = None\n\n        if exists(closure):\n            loss = closure()\n\n        for group in self.param_groups:\n\n            lr = group['lr']\n            beta1, beta2, beta3 = group['betas']\n            weight_decay = group['weight_decay']\n            eps = group['eps']\n            restart_cond = group['restart_cond']\n\n            for p in group['params']:\n                if not exists(p.grad):\n                    continue\n\n                data, grad = p.data, p.grad.data\n                assert not grad.is_sparse\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['prev_grad'] = torch.zeros_like(grad)\n                    state['m'] = torch.zeros_like(grad)\n                    state['v'] = torch.zeros_like(grad)\n                    state['n'] = torch.zeros_like(grad)\n\n                step, m, v, n, prev_grad = state['step'], state['m'], state['v'], state['n'], state['prev_grad']\n\n                if step > 0:\n                    prev_grad = state['prev_grad']\n\n                    # main algorithm\n\n                    m.mul_(1 - beta1).add_(grad, alpha = beta1)\n\n                    grad_diff = grad - prev_grad\n\n                    v.mul_(1 - beta2).add_(grad_diff, alpha = beta2)\n\n                    next_n = (grad + (1 - beta2) * grad_diff) ** 2\n\n                    n.mul_(1 - beta3).add_(next_n, alpha = beta3)\n\n                # bias correction terms\n\n                step += 1\n\n                correct_m, correct_v, correct_n = map(lambda n: 1 / (1 - (1 - n) ** step), (beta1, beta2, beta3))\n\n                # gradient step\n\n                def grad_step_(data, m, v, n):\n                    weighted_step_size = lr / (n * correct_n).sqrt().add_(eps)\n\n                    denom = 1 + weight_decay * lr\n\n                    data.addcmul_(weighted_step_size, (m * correct_m + (1 - beta2) * v * correct_v), value = -1.).div_(denom)\n\n                grad_step_(data, m, v, n)\n\n                # restart condition\n\n                if exists(restart_cond) and restart_cond(state):\n                    m.data.copy_(grad)\n                    v.zero_()\n                    n.data.copy_(grad ** 2)\n\n                    grad_step_(data, m, v, n)\n\n                # set new incremented step\n\n                prev_grad.copy_(grad)\n                state['step'] = step\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:41.416616Z","iopub.execute_input":"2022-11-02T11:12:41.416915Z","iopub.status.idle":"2022-11-02T11:12:41.434053Z","shell.execute_reply.started":"2022-11-02T11:12:41.416886Z","shell.execute_reply":"2022-11-02T11:12:41.433044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_tabnet.metrics import Metric\nfrom sklearn.metrics import (\n    r2_score\n)\nclass r2score(Metric):\n    \"\"\"\n    Mean Absolute Error.\n    \"\"\"\n\n    def __init__(self):\n        self._name = \"neg_r2\"\n        self._maximize = False\n\n    def __call__(self, y_true, y_score):\n        return -r2_score(y_true, y_score)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:41.435213Z","iopub.execute_input":"2022-11-02T11:12:41.436514Z","iopub.status.idle":"2022-11-02T11:12:41.456143Z","shell.execute_reply.started":"2022-11-02T11:12:41.436464Z","shell.execute_reply":"2022-11-02T11:12:41.454913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CCCLoss(torch.nn.Module):\n    def __init__(self, eps=1e-6):\n        super(CCCLoss, self).__init__()\n        self.eps = eps\n\n    def forward(self, y_true, y_hat):\n        y_true_mean = torch.mean(y_true)\n        y_hat_mean = torch.mean(y_hat)\n        y_true_var = torch.var(y_true)\n        y_hat_var = torch.var(y_hat)\n        y_true_std = torch.std(y_true)\n        y_hat_std = torch.std(y_hat)\n        vx = y_true - torch.mean(y_true)\n        vy = y_hat - torch.mean(y_hat)\n        pcc = torch.sum(vx * vy) / (torch.sqrt(torch.sum(vx ** 2) + self.eps) * torch.sqrt(torch.sum(vy ** 2) + self.eps))\n        ccc = (2 * pcc * y_true_std * y_hat_std) / \\\n              (y_true_var + y_hat_var + (y_hat_mean - y_true_mean) ** 2)\n        ccc = 1 - ccc\n        return ccc\nloss_fn = CCCLoss()","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:12:41.457687Z","iopub.execute_input":"2022-11-02T11:12:41.458134Z","iopub.status.idle":"2022-11-02T11:12:41.472284Z","shell.execute_reply.started":"2022-11-02T11:12:41.458105Z","shell.execute_reply":"2022-11-02T11:12:41.471003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\nmax_epochs = 100\ndef objective(trial):\n    alpha = trial.suggest_float(\"alpha\", 1e-5, 1e-1, log=True)\n    \n    \n    n_d= trial.suggest_int(\"n_d\", 1, 10, log=True)\n    n_a= n_d\n    n_steps = trial.suggest_int(\"n_steps\", 3, 10, log=True) #3 < n_steps < 10\n    gamma = trial.suggest_float(\"gamma\", 1, 2, log=True) #1 < gamma< 2\n    n_independent = trial.suggest_int(\"n_independent\", 1, 5, log=True)#1 < n_independent < 5\n    n_shared = trial.suggest_int(\"n_shared\", 1, 5, log=True)#1 < n_shared < 5\n#     momentum = trial.suggest_float(\"n_d\", 0.01, 0.4, log=True)#0.01 < momentum < 0.4\n    \n    lambda_sparse = trial.suggest_float(\"lambda_sparse\", 10e-5, 0.5, log=True)#flaot\n    model = TabNetRegressor(optimizer_fn=Adan,\n                            optimizer_params=dict(lr=0.02), \n                            n_d = n_d,\n                            n_a = n_a, \n                            n_steps = n_steps, \n                            gamma = gamma,\n                            n_independent = n_independent,\n                            n_shared=n_shared,\n                            lambda_sparse=lambda_sparse,\n                            device_name='cuda', \n                            verbose=0\n                           )    \n    model.fit(\n        X_train=X_train, \n        y_train=y_train,\n        eval_set=[(X_train, y_train), (X_valid, y_valid), (X[main_test_index], y[main_test_index])],\n        eval_name=['train', 'valid', 'test'],\n        eval_metric=[r2score],\n        max_epochs=max_epochs,\n        patience=10,\n        batch_size=2048, \n        virtual_batch_size=128,\n        num_workers=0,\n        drop_last=False,\n        loss_fn = loss_fn\n    ) \n    y_pred = model.predict(X[main_test_index])\n    # Report intermediate objective value.\n#     trial.report(y_pred, step)\n    s = correlation_score( y_pred , y[main_test_index]  )\n        # Handle pruning based on the intermediate value.\n    if trial.should_prune():\n        raise optuna.TrialPruned()\n\n    return s\n\nimport logging\nbest_params = {'alpha': 2.877542989655744e-05, \n               'n_d': 10, \n               'n_steps': 3, \n               'gamma': 1.4003405427055062, \n               'n_independent': 1, \n               'n_shared': 5, \n               'lambda_sparse': 0.0006038455293236929}\noptuna.logging.get_logger(\"optuna\").addHandler(logging.StreamHandler(sys.stdout))\nstudy = optuna.create_study(pruner=optuna.pruners.MedianPruner(), direction = 'maximize')\n\nstudy.enqueue_trial(best_params)\nstudy.optimize(objective, n_trials=100)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:13:13.76241Z","iopub.execute_input":"2022-11-02T11:13:13.762778Z","iopub.status.idle":"2022-11-02T11:22:04.238214Z","shell.execute_reply.started":"2022-11-02T11:13:13.762752Z","shell.execute_reply":"2022-11-02T11:22:04.236206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from optuna.visualization import plot_optimization_history\n\nplot_optimization_history(study)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T20:48:23.189919Z","iopub.execute_input":"2022-11-01T20:48:23.191883Z","iopub.status.idle":"2022-11-01T20:48:23.206519Z","shell.execute_reply.started":"2022-11-01T20:48:23.191837Z","shell.execute_reply":"2022-11-01T20:48:23.205289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add_features_df(r, model, 0.0001)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}