{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import PercentFormatter\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:04.631452Z","iopub.execute_input":"2024-11-28T12:17:04.631937Z","iopub.status.idle":"2024-11-28T12:17:04.646738Z","shell.execute_reply.started":"2024-11-28T12:17:04.631886Z","shell.execute_reply":"2024-11-28T12:17:04.645074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_path = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:04.649443Z","iopub.execute_input":"2024-11-28T12:17:04.650347Z","iopub.status.idle":"2024-11-28T12:17:04.658579Z","shell.execute_reply.started":"2024-11-28T12:17:04.650287Z","shell.execute_reply":"2024-11-28T12:17:04.657074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_df = pl.read_csv(f\"{base_path}/features.csv\")\nresponders_df = pl.read_csv(f\"{base_path}/responders.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:04.901349Z","iopub.execute_input":"2024-11-28T12:17:04.901946Z","iopub.status.idle":"2024-11-28T12:17:04.986227Z","shell.execute_reply.started":"2024-11-28T12:17:04.901887Z","shell.execute_reply":"2024-11-28T12:17:04.984792Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pl.read_parquet(f\"{base_path}/test.parquet/date_id=0/part-0.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:05.238086Z","iopub.execute_input":"2024-11-28T12:17:05.238531Z","iopub.status.idle":"2024-11-28T12:17:05.252321Z","shell.execute_reply.started":"2024-11-28T12:17:05.23849Z","shell.execute_reply":"2024-11-28T12:17:05.250742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = pl.read_parquet(f\"{base_path}/lags.parquet/date_id=0/part-0.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:05.612436Z","iopub.execute_input":"2024-11-28T12:17:05.613626Z","iopub.status.idle":"2024-11-28T12:17:05.622278Z","shell.execute_reply.started":"2024-11-28T12:17:05.613577Z","shell.execute_reply":"2024-11-28T12:17:05.620806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition0 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=0/part-0.parquet\")\ntrain_partition0.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:05.988344Z","iopub.execute_input":"2024-11-28T12:17:05.988786Z","iopub.status.idle":"2024-11-28T12:17:06.829223Z","shell.execute_reply.started":"2024-11-28T12:17:05.988749Z","shell.execute_reply":"2024-11-28T12:17:06.827656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get column names of the DataFrame\ncolumns = train_partition0.columns\nprint(columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:06.831541Z","iopub.execute_input":"2024-11-28T12:17:06.832066Z","iopub.status.idle":"2024-11-28T12:17:06.839034Z","shell.execute_reply.started":"2024-11-28T12:17:06.832013Z","shell.execute_reply":"2024-11-28T12:17:06.837486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"columns_to_remove = ['date_id', 'time_id', 'symbol_id','responder_0', 'responder_1', 'responder_2'\n                     ,'responder_3', 'responder_4', 'responder_5', 'responder_7', 'responder_8']\n\n# Drop the columns\ntrain_partition0 = train_partition0.drop(columns_to_remove)\ntrain_partition0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:06.963112Z","iopub.execute_input":"2024-11-28T12:17:06.963503Z","iopub.status.idle":"2024-11-28T12:17:06.98664Z","shell.execute_reply.started":"2024-11-28T12:17:06.963471Z","shell.execute_reply":"2024-11-28T12:17:06.985179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition0\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:07.552152Z","iopub.execute_input":"2024-11-28T12:17:07.552596Z","iopub.status.idle":"2024-11-28T12:17:08.5977Z","shell.execute_reply.started":"2024-11-28T12:17:07.552538Z","shell.execute_reply":"2024-11-28T12:17:08.596212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition1 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=1/part-0.parquet\")\ntrain_partition1 = train_partition1.drop(columns_to_remove)\ntrain_partition1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:08.600174Z","iopub.execute_input":"2024-11-28T12:17:08.600728Z","iopub.status.idle":"2024-11-28T12:17:09.92089Z","shell.execute_reply.started":"2024-11-28T12:17:08.600678Z","shell.execute_reply":"2024-11-28T12:17:09.919761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition1\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:09.922727Z","iopub.execute_input":"2024-11-28T12:17:09.923086Z","iopub.status.idle":"2024-11-28T12:17:10.979539Z","shell.execute_reply.started":"2024-11-28T12:17:09.92305Z","shell.execute_reply":"2024-11-28T12:17:10.977978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition2 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=2/part-0.parquet\")\ntrain_partition2 = train_partition2.drop(columns_to_remove)\ntrain_partition2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:10.981333Z","iopub.execute_input":"2024-11-28T12:17:10.981842Z","iopub.status.idle":"2024-11-28T12:17:12.741737Z","shell.execute_reply.started":"2024-11-28T12:17:10.981796Z","shell.execute_reply":"2024-11-28T12:17:12.740404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition2\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:12.744311Z","iopub.execute_input":"2024-11-28T12:17:12.744699Z","iopub.status.idle":"2024-11-28T12:17:13.887512Z","shell.execute_reply.started":"2024-11-28T12:17:12.744663Z","shell.execute_reply":"2024-11-28T12:17:13.886307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition3 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=3/part-0.parquet\")\ntrain_partition3 = train_partition3.drop(columns_to_remove)\ntrain_partition3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:13.889086Z","iopub.execute_input":"2024-11-28T12:17:13.889466Z","iopub.status.idle":"2024-11-28T12:17:16.116752Z","shell.execute_reply.started":"2024-11-28T12:17:13.889429Z","shell.execute_reply":"2024-11-28T12:17:16.115609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition3\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:16.118514Z","iopub.execute_input":"2024-11-28T12:17:16.119124Z","iopub.status.idle":"2024-11-28T12:17:17.221103Z","shell.execute_reply.started":"2024-11-28T12:17:16.119066Z","shell.execute_reply":"2024-11-28T12:17:17.219798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition4 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=4/part-0.parquet\")\n\ntrain_partition4 = train_partition4.drop(columns_to_remove)\ntrain_partition4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:17.222822Z","iopub.execute_input":"2024-11-28T12:17:17.223646Z","iopub.status.idle":"2024-11-28T12:17:23.062974Z","shell.execute_reply.started":"2024-11-28T12:17:17.223588Z","shell.execute_reply":"2024-11-28T12:17:23.061475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition4\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:23.064598Z","iopub.execute_input":"2024-11-28T12:17:23.065066Z","iopub.status.idle":"2024-11-28T12:17:24.127114Z","shell.execute_reply.started":"2024-11-28T12:17:23.065013Z","shell.execute_reply":"2024-11-28T12:17:24.12565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition5 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=5/part-0.parquet\")\ntrain_partition5 = train_partition5.drop(columns_to_remove)\ntrain_partition5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:24.128805Z","iopub.execute_input":"2024-11-28T12:17:24.129252Z","iopub.status.idle":"2024-11-28T12:17:29.280762Z","shell.execute_reply.started":"2024-11-28T12:17:24.129211Z","shell.execute_reply":"2024-11-28T12:17:29.279636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition5\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:29.284772Z","iopub.execute_input":"2024-11-28T12:17:29.285154Z","iopub.status.idle":"2024-11-28T12:17:30.446448Z","shell.execute_reply.started":"2024-11-28T12:17:29.285118Z","shell.execute_reply":"2024-11-28T12:17:30.445223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition6 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=6/part-0.parquet\")\ntrain_partition6 = train_partition6.drop(columns_to_remove)\ntrain_partition6","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:30.448057Z","iopub.execute_input":"2024-11-28T12:17:30.448443Z","iopub.status.idle":"2024-11-28T12:17:36.963694Z","shell.execute_reply.started":"2024-11-28T12:17:30.448405Z","shell.execute_reply":"2024-11-28T12:17:36.962315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition6\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:36.965215Z","iopub.execute_input":"2024-11-28T12:17:36.965675Z","iopub.status.idle":"2024-11-28T12:17:37.967789Z","shell.execute_reply.started":"2024-11-28T12:17:36.965635Z","shell.execute_reply":"2024-11-28T12:17:37.966517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition7 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=7/part-0.parquet\")\ntrain_partition7 = train_partition7.drop(columns_to_remove)\ntrain_partition7","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:37.969602Z","iopub.execute_input":"2024-11-28T12:17:37.970778Z","iopub.status.idle":"2024-11-28T12:17:44.554092Z","shell.execute_reply.started":"2024-11-28T12:17:37.970718Z","shell.execute_reply":"2024-11-28T12:17:44.552885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition7\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:44.555373Z","iopub.execute_input":"2024-11-28T12:17:44.55574Z","iopub.status.idle":"2024-11-28T12:17:45.601534Z","shell.execute_reply.started":"2024-11-28T12:17:44.555705Z","shell.execute_reply":"2024-11-28T12:17:45.60041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition8 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=8/part-0.parquet\")\ntrain_partition8 = train_partition8.drop(columns_to_remove)\ntrain_partition8","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:45.603479Z","iopub.execute_input":"2024-11-28T12:17:45.603974Z","iopub.status.idle":"2024-11-28T12:17:54.413571Z","shell.execute_reply.started":"2024-11-28T12:17:45.603921Z","shell.execute_reply":"2024-11-28T12:17:54.408403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition8\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:54.419909Z","iopub.execute_input":"2024-11-28T12:17:54.421225Z","iopub.status.idle":"2024-11-28T12:17:56.681401Z","shell.execute_reply.started":"2024-11-28T12:17:54.421146Z","shell.execute_reply":"2024-11-28T12:17:56.679103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_partition9 = pl.read_parquet(f\"{base_path}/train.parquet/partition_id=9/part-0.parquet\")\ntrain_partition9 = train_partition9.drop(columns_to_remove)\ntrain_partition9","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:17:56.684242Z","iopub.execute_input":"2024-11-28T12:17:56.684908Z","iopub.status.idle":"2024-11-28T12:18:02.004297Z","shell.execute_reply.started":"2024-11-28T12:17:56.684803Z","shell.execute_reply":"2024-11-28T12:18:02.002676Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition9\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:18:02.006132Z","iopub.execute_input":"2024-11-28T12:18:02.00653Z","iopub.status.idle":"2024-11-28T12:18:03.237577Z","shell.execute_reply.started":"2024-11-28T12:18:02.006493Z","shell.execute_reply":"2024-11-28T12:18:03.236282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"partitions = [\n    train_partition3, train_partition4, train_partition5, \n    train_partition6, train_partition7, train_partition8, train_partition9\n]\n\ntrain_full = pl.concat(partitions)\ntrain_full","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T12:30:09.457456Z","iopub.execute_input":"2024-11-28T12:30:09.458807Z","iopub.status.idle":"2024-11-28T12:30:09.49289Z","shell.execute_reply.started":"2024-11-28T12:30:09.458759Z","shell.execute_reply":"2024-11-28T12:30:09.49159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\n# Separate features and responders\nfeatures = sample_df.filter(regex='^feature_')\nresponders = sample_df.filter(regex='^responder_')\nweights = sample_df['weight']\n# Convert to numpy arrays for TensorFlow\nX = features.values  # Features for input\n#y = responders.values  # Responders for output\n# Assuming you have a DataFrame y_train with all responders\ny = responders[['responder_6']].values  # Keep only responder_6\nX = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)\ny = np.nan_to_num(y, nan=0.0, posinf=0.0, neginf=0.0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"supervised_usable = (\n    train_partition0\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}