{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":7.594014,"end_time":"2024-10-10T11:58:36.355301","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-10T11:58:28.761287","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport joblib \n\nimport pandas as pd\nimport polars as pl\nimport numpy as np \n\nfrom joblib import Parallel, delayed\nimport gc\n\nimport matplotlib.pyplot as plt\n\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":1.594977,"end_time":"2024-10-10T11:58:33.569648","exception":false,"start_time":"2024-10-10T11:58:31.974671","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-11-10T05:44:39.285024Z","iopub.execute_input":"2024-11-10T05:44:39.285506Z","iopub.status.idle":"2024-11-10T05:44:39.293356Z","shell.execute_reply.started":"2024-11-10T05:44:39.285447Z","shell.execute_reply":"2024-11-10T05:44:39.291352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(date_id_range=None, time_id_range=None, columns=None, return_type='pl'):\n    \"\"\"\n    Load data from Parquet files with optional filtering on date_id, time_id, and selected columns.\n\n    Parameters:\n    - date_id_range (tuple, optional): Range of date_id to filter (start, end). Default is None, which means all dates.\n    - time_id_range (tuple, optional): Range of time_id to filter (start, end). Default is None, which means all times.\n    - columns (list, optional): List of columns to load. Default is None, which means all columns.\n    - return_type (str, optional): Type of data to return ('pl' for Polars DataFrame or 'pd' for Pandas DataFrame). Default is 'pl'.\n\n    Returns:\n    - pl.DataFrame or pd.DataFrame: The filtered data as a Polars or Pandas DataFrame.\n    \"\"\"\n    data_dir = '../input/jane-street-real-time-market-data-forecasting'\n    # Load data using Polars lazy loading (scan_parquet)\n    data = pl.scan_parquet(f\"{data_dir}/train.parquet\")\n\n    # Apply date_id filter if specified\n    if date_id_range is not None:\n        start_date, end_date = date_id_range\n        data = data.filter((pl.col(\"date_id\") >= start_date) & (pl.col(\"date_id\") <= end_date))\n\n    # Apply time_id filter if specified\n    if time_id_range is not None:\n        start_time, end_time = time_id_range\n        data = data.filter((pl.col(\"time_id\") >= start_time) & (pl.col(\"time_id\") <= end_time))\n\n    # Select specific columns if specified\n    if columns is not None:\n        data = data.select(columns)\n\n    # Collect the data to execute the lazy operations\n    if return_type == 'pd':\n        return data.collect().to_pandas()\n    else:\n        return data.collect()\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:44:39.375278Z","iopub.execute_input":"2024-11-10T05:44:39.375810Z","iopub.status.idle":"2024-11-10T05:44:39.387333Z","shell.execute_reply.started":"2024-11-10T05:44:39.375753Z","shell.execute_reply":"2024-11-10T05:44:39.386005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=load_data()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:44:39.389924Z","iopub.execute_input":"2024-11-10T05:44:39.390344Z","iopub.status.idle":"2024-11-10T05:45:31.533958Z","shell.execute_reply.started":"2024-11-10T05:44:39.390302Z","shell.execute_reply":"2024-11-10T05:45:31.528401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.schema\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.539291Z","iopub.execute_input":"2024-11-10T05:45:31.540877Z","iopub.status.idle":"2024-11-10T05:45:31.586416Z","shell.execute_reply.started":"2024-11-10T05:45:31.540762Z","shell.execute_reply":"2024-11-10T05:45:31.580840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.select(pl.col(\"*\").is_null().sum())\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.595757Z","iopub.execute_input":"2024-11-10T05:45:31.596827Z","iopub.status.idle":"2024-11-10T05:45:31.626066Z","shell.execute_reply.started":"2024-11-10T05:45:31.596768Z","shell.execute_reply":"2024-11-10T05:45:31.622238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the total number of rows in the DataFrame\ntotal_rows = data.height\nprint(f'Total number of rows: {total_rows}')\n# Count NA values, add percentage, filter for counts > 0, and sort by decreasing count of NAs\nna_counts = (\n    data.select([pl.col(column).is_null().sum().alias(column) for column in data.columns])\n    .melt()  # Reshape to long format, with 'variable' and 'value' columns\n    .filter(pl.col(\"value\") > 0)  # Filter for columns with NA count > 0\n    .with_columns((pl.col(\"value\") / total_rows * 100).alias(\"percent_missing\"))  # Calculate percentage\n    .sort(by=\"value\", descending=True)  # Sort by NA count in descending order\n)\n\nna_counts","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.629822Z","iopub.execute_input":"2024-11-10T05:45:31.630881Z","iopub.status.idle":"2024-11-10T05:45:31.683592Z","shell.execute_reply.started":"2024-11-10T05:45:31.630719Z","shell.execute_reply":"2024-11-10T05:45:31.679543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract and print only the variable names with missing values\nvariable_names_with_na = (na_counts.select(\"variable\")\n                          .to_series()\n                          .to_list())  # Ensure unique variable names\n\nvariable_names_with_na","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.689688Z","iopub.execute_input":"2024-11-10T05:45:31.692945Z","iopub.status.idle":"2024-11-10T05:45:31.733331Z","shell.execute_reply.started":"2024-11-10T05:45:31.692763Z","shell.execute_reply":"2024-11-10T05:45:31.728730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.columns)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.737969Z","iopub.execute_input":"2024-11-10T05:45:31.738756Z","iopub.status.idle":"2024-11-10T05:45:31.760950Z","shell.execute_reply.started":"2024-11-10T05:45:31.738712Z","shell.execute_reply":"2024-11-10T05:45:31.753593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_columns = [col for col in data.columns if 'responder' in col]\nprint(target_columns)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.765058Z","iopub.execute_input":"2024-11-10T05:45:31.766775Z","iopub.status.idle":"2024-11-10T05:45:31.792145Z","shell.execute_reply.started":"2024-11-10T05:45:31.766690Z","shell.execute_reply":"2024-11-10T05:45:31.785786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"partition_col=['date_id', 'time_id', 'symbol_id', 'weight','partition_id']","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.817968Z","iopub.execute_input":"2024-11-10T05:45:31.819776Z","iopub.status.idle":"2024-11-10T05:45:31.843826Z","shell.execute_reply.started":"2024-11-10T05:45:31.819375Z","shell.execute_reply":"2024-11-10T05:45:31.836949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"retain_cols=variable_names_with_na+target_columns+partition_col","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.852266Z","iopub.execute_input":"2024-11-10T05:45:31.854125Z","iopub.status.idle":"2024-11-10T05:45:31.876550Z","shell.execute_reply.started":"2024-11-10T05:45:31.853987Z","shell.execute_reply":"2024-11-10T05:45:31.869875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.drop([col for col in data.columns if col not in retain_cols])\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.880733Z","iopub.execute_input":"2024-11-10T05:45:31.882544Z","iopub.status.idle":"2024-11-10T05:45:31.938687Z","shell.execute_reply.started":"2024-11-10T05:45:31.882222Z","shell.execute_reply":"2024-11-10T05:45:31.934532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:31.944594Z","iopub.execute_input":"2024-11-10T05:45:31.946592Z","iopub.status.idle":"2024-11-10T05:45:32.539249Z","shell.execute_reply.started":"2024-11-10T05:45:31.946380Z","shell.execute_reply":"2024-11-10T05:45:32.534335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head(50)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:32.547145Z","iopub.execute_input":"2024-11-10T05:45:32.563473Z","iopub.status.idle":"2024-11-10T05:45:32.717525Z","shell.execute_reply.started":"2024-11-10T05:45:32.563265Z","shell.execute_reply":"2024-11-10T05:45:32.709900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_symbols = data.select(pl.col('symbol_id').unique())\n\n# To print the unique values\nprint(unique_symbols)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:32.736801Z","iopub.execute_input":"2024-11-10T05:45:32.749940Z","iopub.status.idle":"2024-11-10T05:45:34.648258Z","shell.execute_reply.started":"2024-11-10T05:45:32.749623Z","shell.execute_reply":"2024-11-10T05:45:34.645447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_time = data.select(pl.col('time_id').unique())\n\n# To print the unique values\nprint(unique_time)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:34.653183Z","iopub.execute_input":"2024-11-10T05:45:34.659307Z","iopub.status.idle":"2024-11-10T05:45:36.353647Z","shell.execute_reply.started":"2024-11-10T05:45:34.658912Z","shell.execute_reply":"2024-11-10T05:45:36.345959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:36.365119Z","iopub.execute_input":"2024-11-10T05:45:36.372213Z","iopub.status.idle":"2024-11-10T05:45:36.733373Z","shell.execute_reply.started":"2024-11-10T05:45:36.371610Z","shell.execute_reply":"2024-11-10T05:45:36.727176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = (\n    data.group_by('date_id')\n    .agg(pl.col('time_id').unique().alias('unique_time_id'))  # Aggregate unique 'time_id'\n)\n\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:36.750761Z","iopub.execute_input":"2024-11-10T05:45:36.756315Z","iopub.status.idle":"2024-11-10T05:45:57.677575Z","shell.execute_reply.started":"2024-11-10T05:45:36.755882Z","shell.execute_reply":"2024-11-10T05:45:57.674549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = (\n    data.group_by('date_id')\n    .agg(pl.col('time_id')\n    .n_unique()\n    .alias('unique_time_id'))\n    .sort('date_id', descending=True)  # Aggregate unique 'time_id'\n)\n\nresult","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:45:57.681924Z","iopub.execute_input":"2024-11-10T05:45:57.684585Z","iopub.status.idle":"2024-11-10T05:46:09.771993Z","shell.execute_reply.started":"2024-11-10T05:45:57.684455Z","shell.execute_reply":"2024-11-10T05:46:09.770554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['unique_time_id'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:09.773792Z","iopub.execute_input":"2024-11-10T05:46:09.774298Z","iopub.status.idle":"2024-11-10T05:46:09.785828Z","shell.execute_reply.started":"2024-11-10T05:46:09.774244Z","shell.execute_reply":"2024-11-10T05:46:09.784577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.filter(pl.col('unique_time_id')==968)['date_id'].min()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:09.787629Z","iopub.execute_input":"2024-11-10T05:46:09.788147Z","iopub.status.idle":"2024-11-10T05:46:09.803302Z","shell.execute_reply.started":"2024-11-10T05:46:09.788090Z","shell.execute_reply":"2024-11-10T05:46:09.802163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(result['date_id'],result['unique_time_id'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:09.804990Z","iopub.execute_input":"2024-11-10T05:46:09.805523Z","iopub.status.idle":"2024-11-10T05:46:10.034324Z","shell.execute_reply.started":"2024-11-10T05:46:09.805462Z","shell.execute_reply":"2024-11-10T05:46:10.033024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = (\n    data.group_by('date_id')\n    .agg(pl.col('symbol_id')\n    .n_unique()\n    .alias('unique_symbols'))\n    .sort('date_id', descending=True)  # Aggregate unique 'time_id'\n)\n\nresult","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:10.036071Z","iopub.execute_input":"2024-11-10T05:46:10.036457Z","iopub.status.idle":"2024-11-10T05:46:20.803211Z","shell.execute_reply.started":"2024-11-10T05:46:10.036417Z","shell.execute_reply":"2024-11-10T05:46:20.801776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(result['date_id'],result['unique_symbols'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:20.805083Z","iopub.execute_input":"2024-11-10T05:46:20.805601Z","iopub.status.idle":"2024-11-10T05:46:21.095207Z","shell.execute_reply.started":"2024-11-10T05:46:20.805545Z","shell.execute_reply":"2024-11-10T05:46:21.093710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['weight'].min(),data['weight'].max(),data['weight'].mean(),data['weight'].std()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:21.103156Z","iopub.execute_input":"2024-11-10T05:46:21.103613Z","iopub.status.idle":"2024-11-10T05:46:21.423709Z","shell.execute_reply.started":"2024-11-10T05:46:21.103570Z","shell.execute_reply":"2024-11-10T05:46:21.422201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by 'date_id' and calculate the mean, median, std, min, and max for 'weight'\ndate_weight_agg = data.group_by('date_id').agg([\n    pl.col('weight').mean().alias('weight_mean'),\n    pl.col('weight').median().alias('weight_median'),\n    pl.col('weight').std().alias('weight_std'),\n    pl.col('weight').min().alias('weight_min'),\n    pl.col('weight').max().alias('weight_max')\n])\n\n# Print the result\ndate_weight_agg","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:21.425951Z","iopub.execute_input":"2024-11-10T05:46:21.426334Z","iopub.status.idle":"2024-11-10T05:46:32.515026Z","shell.execute_reply.started":"2024-11-10T05:46:21.426295Z","shell.execute_reply":"2024-11-10T05:46:32.513528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by 'date_id' and calculate the mean, median, std, min, and max for 'weight'\ndate_time_weight_agg = data.group_by(['date_id','time_id']).agg([\n    pl.col('weight').mean().alias('weight_mean'),\n    pl.col('weight').median().alias('weight_median'),\n    pl.col('weight').std().alias('weight_std'),\n    pl.col('weight').min().alias('weight_min'),\n    pl.col('weight').max().alias('weight_max'),\n    pl.col('symbol_id').n_unique().alias('n_symbols')\n])\n\n# Print the result\ndate_time_weight_agg","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:32.517091Z","iopub.execute_input":"2024-11-10T05:46:32.517634Z","iopub.status.idle":"2024-11-10T05:46:49.308250Z","shell.execute_reply.started":"2024-11-10T05:46:32.517576Z","shell.execute_reply":"2024-11-10T05:46:49.306701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_time_weight_agg = date_time_weight_agg.with_columns(\n    (pl.col('weight_mean') * pl.col('n_symbols')).alias('wt_sum')\n)\n\n# Print the result\ndate_time_weight_agg","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:49.309976Z","iopub.execute_input":"2024-11-10T05:46:49.310527Z","iopub.status.idle":"2024-11-10T05:46:49.336268Z","shell.execute_reply.started":"2024-11-10T05:46:49.310449Z","shell.execute_reply":"2024-11-10T05:46:49.334939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(date_time_weight_agg['wt_sum'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:49.338289Z","iopub.execute_input":"2024-11-10T05:46:49.338893Z","iopub.status.idle":"2024-11-10T05:46:49.674061Z","shell.execute_reply.started":"2024-11-10T05:46:49.338848Z","shell.execute_reply":"2024-11-10T05:46:49.672702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by 'date_id' and calculate the mean, median, std, min, and max for 'weight'\ndate_symbol_weight_agg = data.group_by(['date_id','symbol_id']).agg([\n    pl.col('weight').mean().alias('weight_mean'),\n    pl.col('weight').median().alias('weight_median'),\n    pl.col('weight').std().alias('weight_std'),\n    pl.col('weight').min().alias('weight_min'),\n    pl.col('weight').max().alias('weight_max')\n])\n\n# Print the result\ndate_symbol_weight_agg","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:46:49.675615Z","iopub.execute_input":"2024-11-10T05:46:49.676017Z","iopub.status.idle":"2024-11-10T05:47:03.209351Z","shell.execute_reply.started":"2024-11-10T05:46:49.675977Z","shell.execute_reply":"2024-11-10T05:47:03.208034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Insight:\n- Each symbol has same weight for a particular date at different time ids","metadata":{}},{"cell_type":"code","source":"weight_10=(date_symbol_weight_agg.\n           filter(pl.col(\"symbol_id\") == 10)\n           .select(pl.col('date_id'),pl.col(\"weight_mean\"))\n          .sort(\"date_id\"))\nweight_10","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:03.211280Z","iopub.execute_input":"2024-11-10T05:47:03.211813Z","iopub.status.idle":"2024-11-10T05:47:03.225190Z","shell.execute_reply.started":"2024-11-10T05:47:03.211754Z","shell.execute_reply":"2024-11-10T05:47:03.223830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(weight_10[\"date_id\"],weight_10[\"weight_mean\"])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:03.227177Z","iopub.execute_input":"2024-11-10T05:47:03.227651Z","iopub.status.idle":"2024-11-10T05:47:03.439295Z","shell.execute_reply.started":"2024-11-10T05:47:03.227596Z","shell.execute_reply":"2024-11-10T05:47:03.437880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weight_20=(date_symbol_weight_agg.\n           filter(pl.col(\"symbol_id\") == 20)\n           .select(pl.col('date_id'),pl.col(\"weight_mean\"))\n          .sort(\"date_id\"))\nweight_20","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:03.441170Z","iopub.execute_input":"2024-11-10T05:47:03.441715Z","iopub.status.idle":"2024-11-10T05:47:03.455766Z","shell.execute_reply.started":"2024-11-10T05:47:03.441658Z","shell.execute_reply":"2024-11-10T05:47:03.454199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(weight_10[\"date_id\"],weight_10[\"weight_mean\"])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:03.457827Z","iopub.execute_input":"2024-11-10T05:47:03.458244Z","iopub.status.idle":"2024-11-10T05:47:03.670269Z","shell.execute_reply.started":"2024-11-10T05:47:03.458204Z","shell.execute_reply":"2024-11-10T05:47:03.669009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by 'date_id' and calculate the mean, median, std, min, and max for 'weight'\ndate_symbol_agg = data.group_by(['symbol_id']).agg([\n    pl.col('weight').mean().alias('weight_mean'),\n    pl.col('weight').median().alias('weight_median'),\n    pl.col('weight').std().alias('weight_std'),\n    pl.col('weight').min().alias('weight_min'),\n    pl.col('weight').max().alias('weight_max'),\n    pl.col('weight').n_unique().alias('n_entry'),\n    pl.col('date_id').n_unique().alias('n_date')])\n\n# Print the result\ndate_symbol_agg","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:03.672037Z","iopub.execute_input":"2024-11-10T05:47:03.672412Z","iopub.status.idle":"2024-11-10T05:47:17.731432Z","shell.execute_reply.started":"2024-11-10T05:47:03.672374Z","shell.execute_reply":"2024-11-10T05:47:17.728335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(date_symbol_agg['n_date'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:17.735430Z","iopub.execute_input":"2024-11-10T05:47:17.737296Z","iopub.status.idle":"2024-11-10T05:47:18.174352Z","shell.execute_reply.started":"2024-11-10T05:47:17.737150Z","shell.execute_reply":"2024-11-10T05:47:18.172958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by 'partition_id' and calculate the Nan Count \nNa_count_partitionwise = data.group_by(['partition_id']).agg([\n    pl.all().null_count()])\n\n# Print the result\nNa_count_partitionwise","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:18.176130Z","iopub.execute_input":"2024-11-10T05:47:18.176657Z","iopub.status.idle":"2024-11-10T05:47:34.044188Z","shell.execute_reply.started":"2024-11-10T05:47:18.176599Z","shell.execute_reply":"2024-11-10T05:47:34.041398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Na_count_partitionwise =Na_count_partitionwise.with_columns(\n    pl.sum_horizontal(a for a in Na_count_partitionwise.columns[1:])\n    .alias('Totla_NA'))\nNa_count_partitionwise","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:34.047273Z","iopub.execute_input":"2024-11-10T05:47:34.048590Z","iopub.status.idle":"2024-11-10T05:47:34.067426Z","shell.execute_reply.started":"2024-11-10T05:47:34.048448Z","shell.execute_reply":"2024-11-10T05:47:34.064618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(Na_count_partitionwise['partition_id'],Na_count_partitionwise['Totla_NA'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:47:34.070762Z","iopub.execute_input":"2024-11-10T05:47:34.071651Z","iopub.status.idle":"2024-11-10T05:47:34.573329Z","shell.execute_reply.started":"2024-11-10T05:47:34.071596Z","shell.execute_reply":"2024-11-10T05:47:34.570672Z"},"trusted":true},"execution_count":null,"outputs":[]}]}