{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7122895,"sourceType":"datasetVersion","datasetId":3948965}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What is about ?\n\nExlore the magic postprocessing propsed by the 4th place team: \nhttps://www.kaggle.com/code/raki21/4th-place-magic-postprocessing\n\nIndeed it is real magic ! Apply to submits - and get private score better than current top1. \n\n\nVersions:\n\n    1 Apply Magic Mult = 1.3 to 3th place submits Public 0.548 Private 0.732: get: 0.545, 0.724 - Indeed it works ! Private - better than top1\n    2 Mult = 1.4 Get: public 0.547 , private 0.723 - similar to original - public worsens, but pricate improves \n    3 Mult = 1.5 Get: 0.551, 0.724  \n    \n    4 Take other submit: Pyboost26PlaceAmbrosM Public 0.572 Private 0.748: Got 0.570, 0.731  - it works ! \n    5 Mult = 1.4 : 0.569 , 0.724 - Private - better than top1 \n    6 Mult = 1.5 : 0.568 , 0.721 - Private - better than top1\n    7 Mult = 1.6 : 0.568 , 0.718 - Private - better than top1\n    8 Mult = 1.7 : 0.569 , 0.718 - public is worse than previous\n    ","metadata":{}},{"cell_type":"markdown","source":"# Preparations","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-12-05T12:55:14.496624Z","iopub.execute_input":"2023-12-05T12:55:14.497013Z","iopub.status.idle":"2023-12-05T12:55:14.898941Z","shell.execute_reply.started":"2023-12-05T12:55:14.496962Z","shell.execute_reply":"2023-12-05T12:55:14.898013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load predictions","metadata":{}},{"cell_type":"code","source":"%%time\nlist_df = []\n# fn = '/kaggle/input/open-problems-2-submits-collection/LB548_732_Kowalski3Place_nbV266.csv'\nfn = '/kaggle/input/open-problems-2-submits-collection/LB572_PyboostAmbrosMtscoreReTrainFull_openAfterEnd_nbV1.csv'\nprint(fn)\ndf = pd.read_csv(fn, index_col = 'id')\nprint(df.shape)\ndisplay(df.head(2))\ndf1 = df.copy()\nlist_df.append(df)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:55:14.900924Z","iopub.execute_input":"2023-12-05T12:55:14.901826Z","iopub.status.idle":"2023-12-05T12:55:19.295803Z","shell.execute_reply.started":"2023-12-05T12:55:14.901781Z","shell.execute_reply":"2023-12-05T12:55:19.294432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Magic postprocessing","metadata":{}},{"cell_type":"code","source":"%%time\n# added postprocessing\ndf_stat = pd.DataFrame()\nMULT = 1.7\nfor index, compound_gene_pred in df.iterrows():\n    if index % 100 == 0:\n        print(index)\n    abs_compound_mean = abs(compound_gene_pred).mean()\n\n    compound_gene_pred *= min(abs_compound_mean**0.6, 1)\n    df.loc[index] = compound_gene_pred  \n\n\n    df_stat.loc[index,'abs_compound_mean'] = abs_compound_mean\n    df_stat.loc[index,'min(abs_compound_mean**0.6, 1)'] = min(abs_compound_mean**0.6, 1)\n    \n    \ndf  *= MULT\n\ndisplay(df.head(3))\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:55:19.297276Z","iopub.execute_input":"2023-12-05T12:55:19.297630Z","iopub.status.idle":"2023-12-05T12:55:19.622304Z","shell.execute_reply.started":"2023-12-05T12:55:19.297600Z","shell.execute_reply":"2023-12-05T12:55:19.621025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save magic submit","metadata":{}},{"cell_type":"code","source":"%%time\ndf.to_csv('submission_4thMagic_Mult1d7_to_Pyboost26PlaceAmbrosM_572_748.csv') \n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:55:19.623757Z","iopub.execute_input":"2023-12-05T12:55:19.624115Z","iopub.status.idle":"2023-12-05T12:55:26.051184Z","shell.execute_reply.started":"2023-12-05T12:55:19.624083Z","shell.execute_reply":"2023-12-05T12:55:26.049603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyse magic","metadata":{}},{"cell_type":"code","source":"df_stat.to_csv('df_stat.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-05T13:15:06.285956Z","iopub.execute_input":"2023-12-05T13:15:06.286454Z","iopub.status.idle":"2023-12-05T13:15:06.296508Z","shell.execute_reply.started":"2023-12-05T13:15:06.286415Z","shell.execute_reply":"2023-12-05T13:15:06.295417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndisplay(df_stat)\ndisplay(df_stat.describe() )\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfor col in df_stat.columns:\n    plt.plot(df_stat[col], label = col)\nplt.grid()\nplt.legend(fontsize = 20)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:55:26.054370Z","iopub.execute_input":"2023-12-05T12:55:26.055391Z","iopub.status.idle":"2023-12-05T12:55:26.863684Z","shell.execute_reply.started":"2023-12-05T12:55:26.055343Z","shell.execute_reply":"2023-12-05T12:55:26.862409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_stat[df_stat.columns[1]] , bins = 30 )\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:55:26.865375Z","iopub.execute_input":"2023-12-05T12:55:26.865627Z","iopub.status.idle":"2023-12-05T12:55:27.054303Z","shell.execute_reply.started":"2023-12-05T12:55:26.865597Z","shell.execute_reply":"2023-12-05T12:55:27.053262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist( df1.values.ravel(), bins = 100 )\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:55:27.055710Z","iopub.execute_input":"2023-12-05T12:55:27.056098Z","iopub.status.idle":"2023-12-05T12:55:27.379801Z","shell.execute_reply.started":"2023-12-05T12:55:27.056062Z","shell.execute_reply":"2023-12-05T12:55:27.378170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('Original submit:')\ndisplay( df1.iloc[0,:].describe() )\nprint('Magic submit:')\ndisplay( df.iloc[0,:].describe() )\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:55:27.381775Z","iopub.execute_input":"2023-12-05T12:55:27.382204Z","iopub.status.idle":"2023-12-05T12:55:27.406620Z","shell.execute_reply.started":"2023-12-05T12:55:27.382174Z","shell.execute_reply":"2023-12-05T12:55:27.405242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stat.columns","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:57:49.074754Z","iopub.execute_input":"2023-12-05T12:57:49.076158Z","iopub.status.idle":"2023-12-05T12:57:49.085055Z","shell.execute_reply.started":"2023-12-05T12:57:49.076076Z","shell.execute_reply":"2023-12-05T12:57:49.083977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(10):\n    print(i, 'abs_compound_mean: ',  df_stat.iloc[i,0], 'min(abs_compound_mean**0.6, 1): ', df_stat.iloc[i,1],  )\n    v = df1.iloc[i,:].values\n    plt.plot(v , '.',  label = 'original')\n    v = df.iloc[i,:].values\n    plt.plot(v ,'.', label = 'transformed')\n    plt.grid()\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:59:56.864463Z","iopub.execute_input":"2023-12-05T12:59:56.864796Z","iopub.status.idle":"2023-12-05T12:59:59.520703Z","shell.execute_reply.started":"2023-12-05T12:59:56.864767Z","shell.execute_reply":"2023-12-05T12:59:59.519314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(10):\n    print(i, 'abs_compound_mean: ',  df_stat.iloc[i,0], 'min(abs_compound_mean**0.6, 1): ', df_stat.iloc[i,1],  )\n    v = df1.iloc[i,:].values\n    plt.hist(v , bins = 100 )\n    plt.grid()\n    plt.show()    \n    v = df.iloc[i,:].values\n    plt.hist(v , bins = 100 )\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T13:04:02.133197Z","iopub.execute_input":"2023-12-05T13:04:02.134651Z","iopub.status.idle":"2023-12-05T13:04:07.849860Z","shell.execute_reply.started":"2023-12-05T13:04:02.134608Z","shell.execute_reply":"2023-12-05T13:04:07.848273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1.7*0.58\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T13:20:05.144193Z","iopub.execute_input":"2023-12-05T13:20:05.144984Z","iopub.status.idle":"2023-12-05T13:20:05.152749Z","shell.execute_reply.started":"2023-12-05T13:20:05.144947Z","shell.execute_reply":"2023-12-05T13:20:05.151591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}