{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7851794,"sourceType":"datasetVersion","datasetId":4604762}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Analysis","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-15T12:34:15.340946Z","iopub.execute_input":"2024-03-15T12:34:15.341274Z","iopub.status.idle":"2024-03-15T12:34:15.373098Z","shell.execute_reply.started":"2024-03-15T12:34:15.341246Z","shell.execute_reply":"2024-03-15T12:34:15.371816Z"}}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport glob","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.323885Z","iopub.execute_input":"2024-03-15T14:32:03.324240Z","iopub.status.idle":"2024-03-15T14:32:03.663395Z","shell.execute_reply.started":"2024-03-15T14:32:03.324216Z","shell.execute_reply":"2024-03-15T14:32:03.662349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_csv_dict = dict()\nfor i in range(5):\n    pred_csv_dict[f'fold_{i}'] = f'/kaggle/input/oof-enis2403087-2/pred_df_f{i}-stage_2.csv'\n\npred_csv_dict","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.667881Z","iopub.execute_input":"2024-03-15T14:32:03.668119Z","iopub.status.idle":"2024-03-15T14:32:03.676707Z","shell.execute_reply.started":"2024-03-15T14:32:03.668096Z","shell.execute_reply":"2024-03-15T14:32:03.675364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df_dict = dict()\nfor key, file_path in pred_csv_dict.items():\n    pred_df_dict[key] = pd.read_csv(file_path)\n    pred_df_dict[key]['fold'] = key","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.677866Z","iopub.execute_input":"2024-03-15T14:32:03.678112Z","iopub.status.idle":"2024-03-15T14:32:03.736913Z","shell.execute_reply.started":"2024-03-15T14:32:03.678090Z","shell.execute_reply":"2024-03-15T14:32:03.735662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df_dict['fold_0'].head()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.740514Z","iopub.execute_input":"2024-03-15T14:32:03.740878Z","iopub.status.idle":"2024-03-15T14:32:03.765984Z","shell.execute_reply.started":"2024-03-15T14:32:03.740844Z","shell.execute_reply":"2024-03-15T14:32:03.764682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(pred_df_dict['fold_0'].columns)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.767172Z","iopub.execute_input":"2024-03-15T14:32:03.767844Z","iopub.status.idle":"2024-03-15T14:32:03.773058Z","shell.execute_reply.started":"2024-03-15T14:32:03.767815Z","shell.execute_reply":"2024-03-15T14:32:03.772390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic","metadata":{}},{"cell_type":"code","source":"pred_df = pd.concat(pred_df_dict.values(), axis=0)\nprint(pred_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.774066Z","iopub.execute_input":"2024-03-15T14:32:03.774283Z","iopub.status.idle":"2024-03-15T14:32:03.788166Z","shell.execute_reply.started":"2024-03-15T14:32:03.774264Z","shell.execute_reply":"2024-03-15T14:32:03.787472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_cols = ['seizure', 'lpd', 'gpd', 'lrda', 'grda', 'other']","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.789061Z","iopub.execute_input":"2024-03-15T14:32:03.789435Z","iopub.status.idle":"2024-03-15T14:32:03.799742Z","shell.execute_reply.started":"2024-03-15T14:32:03.789412Z","shell.execute_reply":"2024-03-15T14:32:03.797677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === basic info ===\naag_dict = {\n        'eeg_id': 'nunique',\n        'patient_id': 'nunique',\n        'fold': 'count'\n    }\npred_df.groupby('fold').agg(aag_dict)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.800814Z","iopub.execute_input":"2024-03-15T14:32:03.801244Z","iopub.status.idle":"2024-03-15T14:32:03.819763Z","shell.execute_reply.started":"2024-03-15T14:32:03.801221Z","shell.execute_reply":"2024-03-15T14:32:03.818851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === total_evaluators ===\npred_df.groupby('fold').agg({'total_evaluators': ['count', 'mean', 'std', 'min', 'max', 'median']})","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.821083Z","iopub.execute_input":"2024-03-15T14:32:03.822114Z","iopub.status.idle":"2024-03-15T14:32:03.840214Z","shell.execute_reply.started":"2024-03-15T14:32:03.822085Z","shell.execute_reply":"2024-03-15T14:32:03.839257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === target ===\npred_df.groupby('fold')['target'].value_counts().unstack()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.841426Z","iopub.execute_input":"2024-03-15T14:32:03.841728Z","iopub.status.idle":"2024-03-15T14:32:03.858139Z","shell.execute_reply.started":"2024-03-15T14:32:03.841703Z","shell.execute_reply":"2024-03-15T14:32:03.857045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === GT ===\nfor label in label_cols:\n    print(f'------------- {label} ---------------')\n    print(pred_df.groupby('fold').agg({f'{label}_vote': ['mean', 'std', 'min', 'max', 'median']}))","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.859229Z","iopub.execute_input":"2024-03-15T14:32:03.860130Z","iopub.status.idle":"2024-03-15T14:32:03.905604Z","shell.execute_reply.started":"2024-03-15T14:32:03.860093Z","shell.execute_reply":"2024-03-15T14:32:03.904289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label in label_cols:\n    print(f\"------------------- {label} -------------------\")\n    # group by fold, and calculate the MSE\n    print(pred_df.groupby('fold').apply(lambda x: ((x[f'{label}_vote'] - x[f'pred_{label}_vote']) ** 2).mean(), include_groups=False))","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.907167Z","iopub.execute_input":"2024-03-15T14:32:03.907445Z","iopub.status.idle":"2024-03-15T14:32:03.946186Z","shell.execute_reply.started":"2024-03-15T14:32:03.907422Z","shell.execute_reply":"2024-03-15T14:32:03.944910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## analysis of samples","metadata":{}},{"cell_type":"code","source":"import numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:32:03.949876Z","iopub.execute_input":"2024-03-15T14:32:03.950154Z","iopub.status.idle":"2024-03-15T14:32:03.956163Z","shell.execute_reply.started":"2024-03-15T14:32:03.950131Z","shell.execute_reply":"2024-03-15T14:32:03.954850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df['mse'] = np.mean((pred_df[[f'{label}_vote' for label in label_cols]].values - pred_df[[f'pred_{label}_vote' for label in label_cols]].values) ** 2, axis=1)\nprint(pred_df['mse'].describe())","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:33:48.342843Z","iopub.execute_input":"2024-03-15T14:33:48.343226Z","iopub.status.idle":"2024-03-15T14:33:48.357243Z","shell.execute_reply.started":"2024-03-15T14:33:48.343184Z","shell.execute_reply":"2024-03-15T14:33:48.355782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pred_df.sort_values(by='mse', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:33:53.093659Z","iopub.execute_input":"2024-03-15T14:33:53.094075Z","iopub.status.idle":"2024-03-15T14:33:53.107820Z","shell.execute_reply.started":"2024-03-15T14:33:53.094041Z","shell.execute_reply":"2024-03-15T14:33:53.105103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df.groupby('fold')['mse'].plot(kind='hist', bins=5, alpha=0.5, legend=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:33:57.206409Z","iopub.execute_input":"2024-03-15T14:33:57.206880Z","iopub.status.idle":"2024-03-15T14:33:57.559216Z","shell.execute_reply.started":"2024-03-15T14:33:57.206856Z","shell.execute_reply":"2024-03-15T14:33:57.557750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df[['fold', 'target', 'mse'] + [f'{label}_vote' for label in label_cols] + [f'pred_{label}_vote' for label in label_cols]].head(10)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:35:19.195637Z","iopub.execute_input":"2024-03-15T14:35:19.196138Z","iopub.status.idle":"2024-03-15T14:35:19.223725Z","shell.execute_reply.started":"2024-03-15T14:35:19.196099Z","shell.execute_reply":"2024-03-15T14:35:19.222182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n in [10, 100, 200, 500, 1000]:\n    print('\\n\\n\\n------------ TOP %d ------------' % n)\n    print(pred_df.head(n)['target'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:35:26.223094Z","iopub.execute_input":"2024-03-15T14:35:26.223624Z","iopub.status.idle":"2024-03-15T14:35:26.237541Z","shell.execute_reply.started":"2024-03-15T14:35:26.223584Z","shell.execute_reply":"2024-03-15T14:35:26.235744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx_2_label = {i: label for i, label in enumerate(label_cols)}\npred_df['pred_target'] = pred_df[[f'pred_{label}_vote' for label in label_cols]].values.argmax(axis=1)\npred_df['pred_target'] = pred_df['pred_target'].map(idx_2_label)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:35:44.343943Z","iopub.execute_input":"2024-03-15T14:35:44.344405Z","iopub.status.idle":"2024-03-15T14:35:44.357681Z","shell.execute_reply.started":"2024-03-15T14:35:44.344369Z","shell.execute_reply":"2024-03-15T14:35:44.356227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n in [10, 100, 500, 1000, 4000]:\n    print('\\n\\n\\n------------ TOP %d ------------' % n)\n    temp_df = pred_df.head(n).reset_index(drop=True)\n    temp_df['target'] = temp_df['target'].str.lower()\n    # all use lower case\n    print(temp_df[temp_df['target'] != temp_df['pred_target']][['target', 'pred_target']].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:36:48.743698Z","iopub.execute_input":"2024-03-15T14:36:48.744024Z","iopub.status.idle":"2024-03-15T14:36:48.777418Z","shell.execute_reply.started":"2024-03-15T14:36:48.744000Z","shell.execute_reply":"2024-03-15T14:36:48.776061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n in [10, 100, 500, 1000, 4000]:\n    print('\\n\\n\\n------------ TOP %d ------------' % n)\n    temp_df = pred_df.head(n).reset_index(drop=True)\n    temp_df['target'] = temp_df['target'].str.lower()\n    # all use lower case\n    print(temp_df[temp_df['target'] != temp_df['pred_target']][['fold', 'target', 'pred_target']].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:35:59.875875Z","iopub.execute_input":"2024-03-15T14:35:59.876343Z","iopub.status.idle":"2024-03-15T14:35:59.917961Z","shell.execute_reply.started":"2024-03-15T14:35:59.876307Z","shell.execute_reply":"2024-03-15T14:35:59.916373Z"},"trusted":true},"execution_count":null,"outputs":[]}]}