{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"},{"sourceId":8875593,"sourceType":"datasetVersion","datasetId":5328625}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/belka-train-submit/modules')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-04T22:49:38.967020Z","iopub.execute_input":"2024-07-04T22:49:38.967383Z","iopub.status.idle":"2024-07-04T22:49:38.978774Z","shell.execute_reply.started":"2024-07-04T22:49:38.967355Z","shell.execute_reply":"2024-07-04T22:49:38.977941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy data files to match location on gcp.\nimport shutil\nshutil.copy('/kaggle/input/belka-train-submit/blocks-3-pca.parquet', './blocks-3-pca.parquet')\nshutil.copy('/kaggle/input/belka-train-submit/mols.parquet', './mols.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-04T22:49:38.980360Z","iopub.execute_input":"2024-07-04T22:49:38.981308Z","iopub.status.idle":"2024-07-04T22:49:39.478037Z","shell.execute_reply.started":"2024-07-04T22:49:38.981281Z","shell.execute_reply":"2024-07-04T22:49:39.476967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy model files to match location on gcp.\n# shutil.copy('/kaggle/input/belka-train-submit/lg-all-d50.state', './lg-all-d50.state')\n# shutil.copy('/kaggle/input/belka-train-submit/lg-all-d50-opt.state', './lg-all-d50-opt.state')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from modules.train import train\nfrom modules.utils import gcp, kaggle\nimport os, torch\n\nuseprior = True\n\noptions = {\n    'epochs': 10,\n    'train_batch_size': 32,\n    'lr': 0.001,\n    'momentum': 0.9,\n    'dropout': 50,\n    'n_rows': 'all',\n    'print_batches': 5000,\n    'network': 'md',\n    'num_splits': 25\n}\n\nrun_name = 'md-all-e8-d50'\n#os.remove(f'out/net/{run_name}.state')\n\n# 1e done\n\n# adam\n# loss rate\n# pca 95\n# batch size\n\n# baseline md 500K 1e rebalance0 dropout50 pca90 = 0.174\n# rebalance10 dropout50 = 0.298\n# rebalance25 dropout50 = 0.293\n# rebalance10 dropout25 = 0.28\n# rebalance10 dropout50 mom80 = 0.293\n# rebalance10 dropout50 pca99 = 0.28\n# rebalance10 dropout50 pca95 = 0.287\n# rebalance10 dropout50 pca95 fixrebalance = 0.275\n\n# strategy \n# lg model (3e has 0.43)\n# rebalance10 \n# dropout50\n# pca90\n\nif gcp():\n    os.system('gsutil cp gs://kaggle-417721/blocks-3-pca.parquet blocks-3-pca.parquet')\n    os.system('gsutil cp gs://kaggle-417721/mols.parquet mols.parquet')\n    os.system(f'gsutil cp gs://kaggle-417721/{run_name}.state {run_name}.state')\n    os.system(f'gsutil cp gs://kaggle-417721/{run_name}-opt.state {run_name}-opt.state')\n    indir = '.' \n    save_folder = '.'\nelif kaggle():\n    indir = '.' \n    save_folder = '.'\nelse:\n    indir = 'out/train/train/'\n    save_folder = 'out/net/'\n\nnet, labels, scores = train(\n    indir = indir,\n    options = options,\n    print_batches = options['print_batches'],\n    save_folder = save_folder,\n    model_load_path = f'{save_folder}/{run_name}',\n    save_name = run_name\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T22:49:39.479833Z","iopub.execute_input":"2024-07-04T22:49:39.480171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from modules.datasets import get_loader\nfrom modules.train import run_val, get_model_optimizer\nfrom modules.utils import gcp, dircreate, kaggle, cloud\nfrom datetime import datetime\nimport pandas as pd\nimport polars as pl\nimport torch, os\nimport numpy as np\n\nif gcp():\n    os.system(f'gsutil cp gs://kaggle-417721/{modelfile}.state {modelfile}.state')\n    os.system(f'gsutil cp gs://kaggle-417721/blocks-3-pca.parquet blocks-3-pca.parquet')\n    os.system('gsutil cp gs://kaggle-417721/test.zip test.zip')\n    os.system('unzip test.zip')\n    mols = pl.read_parquet('test/mols.parquet')\n    blocks = pl.read_parquet('blocks-3-pca.parquet')\n    datafolder = 'test/'\n    load_path = run_name\nelif kaggle():\n    mols = pl.read_parquet('/kaggle/input/belka-train-submit/test/test/mols.parquet')\n    blocks = pl.read_parquet('/kaggle/input/belka-train-submit/blocks-3-pca.parquet')\n    datafolder = '/kaggle/input/belka-train-submit/test/test/'\n    load_path = run_name\nelse:    \n    mols = pl.read_parquet('out/test/mols.parquet')\n    blocks = pl.read_parquet('out/blocks-3-pca.parquet')\n    datafolder = 'out/test/'\n    load_path = f'out/net/{run_name}'\n\nnet, optimizer = get_model_optimizer(\n    options, mols, blocks, \n    load_path = load_path, \n    load_optimizer = False\n)\n\n# run test to get the actual submission.\nmolecule_ids, labels, scores = run_val(\n    get_loader(datafolder, mols = mols, blocks = blocks, options = options, submit = True), \n    net, options\n)\n\nprint('build submission')\ndel labels\nsubmission = []\nfor protein_name in ['sEH', 'BRD4', 'HSA']:\n\n    results = pl.from_pandas(pd.DataFrame({\n        'molecule_id': molecule_ids,\n        'binds': scores[protein_name]\n    })).with_columns(pl.col('molecule_id').cast(pl.Float32))\n    \n    inputs = pl.read_parquet(\n        f'{datafolder}/test-{protein_name}-idsonly.parquet',\n    ).with_columns(pl.col('molecule_id').cast(pl.Float32))\n    inputs = inputs.join(results, on = 'molecule_id', how = 'left')\n    if inputs.null_count()['binds'][0] > 0:\n        raise Exception(f'{inputs.null_count()[\"binds\"][0]:,.0f} nulls after join.')        \n    \n    inputs = inputs .drop('molecule_id')\n\n    isubmission = inputs.select(['id', 'binds'])\n    isubmission = isubmission.select(['id', 'binds'])\n\n    submission.append(isubmission.to_pandas())\n\n    del isubmission, inputs, results, protein_name\n\nsubmission = pd.concat(submission).sort_values('id')\nsubmission['binds'] = np.round(submission['binds'], 3)\n\nif submission.shape[0] != 1674896:\n    raise Exception(f'Submission must have 1674896 rows, found: {submission.shape[0]}')\n\nif cloud():\n    submitfile = f'{run_name}.parquet' if gcp() else 'submission.parquet'\n    submission.to_parquet(submitfile, index = False)\n    if gcp():\n        os.system(f'gsutil cp {submitfile} gs://kaggle-417721/{submitfile}')\nelse:\n    dircreate('out/submit/')\n    submitfile = f'out/submit/{run_name}.parquet'\n    submission.to_parquet(submitfile, index = False)\n\nprint('done!')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove temp files so they don't get confused with output.\nos.remove('./blocks-3-pca.parquet')\nos.remove('./mols.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}