{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20604,"databundleVersionId":1357052,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nimport torch.nn as nn\nimport copy\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pydicom\nimport os\nimport xgboost as xgb\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom collections import OrderedDict\nfrom tqdm import tqdm\nimport pickle\nfrom skimage.transform import resize\nfrom types import SimpleNamespace\nimport torch.nn.functional as F\nimport random\nimport math\nimport torch.optim as optim\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:44:52.145335Z","iopub.execute_input":"2025-07-06T14:44:52.145518Z","iopub.status.idle":"2025-07-06T14:45:00.752130Z","shell.execute_reply.started":"2025-07-06T14:44:52.145501Z","shell.execute_reply":"2025-07-06T14:45:00.751276Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Visualization","metadata":{}},{"cell_type":"code","source":"base_path = \"/kaggle/input/osic-pulmonary-fibrosis-progression/\"\n\ntrain = pd.read_csv(base_path + \"train.csv\")\ntest = pd.read_csv(base_path + \"test.csv\")\nsample_submission = pd.read_csv(base_path + \"sample_submission.csv\")\n\nprint(\"training\")\nprint(train.head())\nprint(\"-\" * 90)\nprint(\"testing\")\nprint(test.head())\nprint(\"-\" * 90)\nprint(\"sample submission\")\nprint(sample_submission.head())","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:00.753293Z","iopub.execute_input":"2025-07-06T14:45:00.753646Z","iopub.status.idle":"2025-07-06T14:45:00.804978Z","shell.execute_reply.started":"2025-07-06T14:45:00.753627Z","shell.execute_reply":"2025-07-06T14:45:00.804120Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(train['FVC'], kde=True)\nplt.title(\"Distribution of FVC\")\nplt.show()\n\nsns.histplot(train['Age'], kde=True)\nplt.title(\"Age distribution\")\nplt.show()\n\nsns.countplot(data=train, x='Sex')\nplt.title(\"Sex distribution\")\nplt.show()\n\nsns.countplot(data=train, x='SmokingStatus')\nplt.title(\"SmokingStatus\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:00.805751Z","iopub.execute_input":"2025-07-06T14:45:00.806054Z","iopub.status.idle":"2025-07-06T14:45:01.610202Z","shell.execute_reply.started":"2025-07-06T14:45:00.806027Z","shell.execute_reply":"2025-07-06T14:45:01.609322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_patient = train[train[\"Patient\"] == train[\"Patient\"].iloc[0]]\nprint(sample_patient)\nplt.plot(sample_patient[\"Weeks\"], sample_patient[\"FVC\"])\nplt.xlabel(\"Weeks\")\nplt.ylabel(\"FVC\")\nplt.show()\n\n\nsample_patient = train[train[\"Patient\"] == train[\"Patient\"].iloc[9]]\nprint(sample_patient)\nplt.plot(sample_patient[\"Weeks\"], sample_patient[\"FVC\"])\nplt.xlabel(\"Weeks\")\nplt.ylabel(\"FVC\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:01.611748Z","iopub.execute_input":"2025-07-06T14:45:01.612003Z","iopub.status.idle":"2025-07-06T14:45:01.876919Z","shell.execute_reply.started":"2025-07-06T14:45:01.611985Z","shell.execute_reply":"2025-07-06T14:45:01.875999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_patient = train[\"Patient\"].iloc[0]\npath = os.path.join(base_path + \"train\", example_patient)\nfiles = sorted(os.listdir(path))\n\ndcm = pydicom.dcmread(os.path.join(path, files[len(files)//2]))\nplt.imshow(dcm.pixel_array, cmap=\"gray\")\nplt.title(f\"CT Scan of {example_patient}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:01.877793Z","iopub.execute_input":"2025-07-06T14:45:01.878067Z","iopub.status.idle":"2025-07-06T14:45:02.170237Z","shell.execute_reply.started":"2025-07-06T14:45:01.878048Z","shell.execute_reply":"2025-07-06T14:45:02.169342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set a clean style\nsns.set(style=\"whitegrid\", palette=\"muted\")\n\n# Set the figure size\nplt.figure(figsize=(18, 5))\n\n# 1. FVC vs Sex\nplt.subplot(1, 3, 1)\nsns.boxplot(data=train, x='Sex', y='FVC')\nplt.title('FVC by Sex')\n\n# 2. FVC vs Age (we'll use scatterplot here)\nplt.subplot(1, 3, 2)\nsns.scatterplot(data=train, x='Age', y='FVC', alpha=0.4)\nplt.title('FVC vs Age')\n\n# 3. FVC vs SmokingStatus\nplt.subplot(1, 3, 3)\nsns.boxplot(data=train, x='SmokingStatus', y='FVC')\nplt.xticks(rotation=20)\nplt.title('FVC by Smoking Status')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:02.170977Z","iopub.execute_input":"2025-07-06T14:45:02.171169Z","iopub.status.idle":"2025-07-06T14:45:02.887661Z","shell.execute_reply.started":"2025-07-06T14:45:02.171154Z","shell.execute_reply":"2025-07-06T14:45:02.886852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set a clean style\nsns.set(style=\"whitegrid\", palette=\"muted\")\n\n# Set the figure size\nplt.figure(figsize=(18, 5))\n\n# 1. Percent by Sex\nplt.subplot(1, 3, 1)\nsns.boxplot(data=train, x='Sex', y='Percent')\nplt.title('Percent by Sex')\n\n# 2. Percent vs Age\nplt.subplot(1, 3, 2)\nsns.scatterplot(data=train, x='Age', y='Percent', alpha=0.4)\nplt.title('Percent vs Age')\n\n# 3. Percent by SmokingStatus\nplt.subplot(1, 3, 3)\nsns.boxplot(data=train, x='SmokingStatus', y='Percent')\nplt.xticks(rotation=20)\nplt.title('Percent by Smoking Status')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:02.888491Z","iopub.execute_input":"2025-07-06T14:45:02.888734Z","iopub.status.idle":"2025-07-06T14:45:03.536746Z","shell.execute_reply.started":"2025-07-06T14:45:02.888714Z","shell.execute_reply":"2025-07-06T14:45:03.535927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(6, 5))\nsns.scatterplot(data=train, x='FVC', y='Percent', alpha=0.5)\nsns.regplot(data=train, x='FVC', y='Percent', scatter=False, color='red', label='Trend')\nplt.title('Relation between FVC and Percent')\nplt.xlabel('FVC (ml)')\nplt.ylabel('Percent (%)')\nplt.legend()\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:03.537855Z","iopub.execute_input":"2025-07-06T14:45:03.538148Z","iopub.status.idle":"2025-07-06T14:45:04.127290Z","shell.execute_reply.started":"2025-07-06T14:45:03.538128Z","shell.execute_reply":"2025-07-06T14:45:04.126515Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"def generate_pairwise_fvc_dataset(df):\n    df = df.dropna()\n    df = df.sort_values(['Patient', 'Weeks'])\n\n    all_rows = []\n\n    for patient_id, group in df.groupby(\"Patient\"):\n        group = group.reset_index(drop=True)\n        n = len(group)\n\n        for i in range(n):\n            for j in range(n):\n                if i == j:\n                    continue\n                row_i = group.iloc[i]\n                row_j = group.iloc[j]\n                all_rows.append({\n                    'Patient_ID': row_i['Patient'],\n                    'Reading_week': row_i['Weeks'],\n                    'target_week': row_j['Weeks'],\n                    'Weeks_diff': row_j['Weeks'] - row_i['Weeks'],\n                    'Curr_FVC': row_i['FVC'],\n                    'Target_FVC': row_j['FVC'],\n                    'Percent': row_i['Percent'],\n                    'Age': row_i['Age'],\n                    'Sex': row_i['Sex'],\n                    'SmokingStatus': row_i['SmokingStatus'],\n                })\n\n    return pd.DataFrame(all_rows)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:04.128179Z","iopub.execute_input":"2025-07-06T14:45:04.128441Z","iopub.status.idle":"2025-07-06T14:45:04.134651Z","shell.execute_reply.started":"2025-07-06T14:45:04.128423Z","shell.execute_reply":"2025-07-06T14:45:04.133900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load original data and generate pairs\ndf = pd.read_csv(base_path + 'train.csv')\npairwise_df = generate_pairwise_fvc_dataset(df)\nprint(pairwise_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:04.137830Z","iopub.execute_input":"2025-07-06T14:45:04.138134Z","iopub.status.idle":"2025-07-06T14:45:05.501712Z","shell.execute_reply.started":"2025-07-06T14:45:04.138107Z","shell.execute_reply":"2025-07-06T14:45:05.500926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Encode categorical variables\nle_sex = LabelEncoder().fit(pairwise_df['Sex'])\nle_smoke = LabelEncoder().fit(pairwise_df['SmokingStatus'])\n\npairwise_df['Sex'] = le_sex.transform(pairwise_df['Sex'])\npairwise_df['SmokingStatus'] = le_smoke.transform(pairwise_df['SmokingStatus'])\n\n# Select features and target\nfeatures = ['Reading_week', 'target_week', 'Weeks_diff', 'Curr_FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus']\ntarget = 'Target_FVC'\n\nX = pairwise_df[features]\ny_processed = pairwise_df[target]\n\n# Normalize continuous features\nscaler = StandardScaler()\nx_processed = scaler.fit_transform(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.502472Z","iopub.execute_input":"2025-07-06T14:45:05.502681Z","iopub.status.idle":"2025-07-06T14:45:05.531278Z","shell.execute_reply.started":"2025-07-06T14:45:05.502664Z","shell.execute_reply":"2025-07-06T14:45:05.530718Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# AdaBoost","metadata":{}},{"cell_type":"code","source":"def get_model(x, y):\n    seed = random.randint(1, 1000)\n    model = xgb.XGBRegressor(\n        objective='reg:squarederror',\n        n_estimators=1000,\n        max_depth=10,\n        learning_rate=0.5,\n        random_state=seed\n\n    )\n    model.fit(x, y)\n    return model, seed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.532013Z","iopub.execute_input":"2025-07-06T14:45:05.532223Z","iopub.status.idle":"2025-07-06T14:45:05.536325Z","shell.execute_reply.started":"2025-07-06T14:45:05.532198Z","shell.execute_reply":"2025-07-06T14:45:05.535678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def adaBoost(k, x, y, thre):\n    m = 0\n    models = []\n    weights = []\n    maes = []\n    seeds = []\n    while m < k:\n        split_seed = random.randint(1, 1000)\n        x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=split_seed)\n        model, model_seed = get_model(x_train, y_train)\n        y_pred = model.predict(x_val)\n        mae = mean_absolute_error(y_val, y_pred)\n        if mae < thre:\n            m += 1\n            w = math.log((thre * 2 - mae) / mae)\n            models.append(model)\n            weights.append(w)\n            maes.append(mae)\n            seeds.append([split_seed, model_seed])\n\n    return models, weights, maes, seeds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.537002Z","iopub.execute_input":"2025-07-06T14:45:05.537215Z","iopub.status.idle":"2025-07-06T14:45:05.555406Z","shell.execute_reply.started":"2025-07-06T14:45:05.537199Z","shell.execute_reply":"2025-07-06T14:45:05.554859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# x_data = x_processed\n# y_data = y_processed\n\n# models, weights, maes, seeds = adaBoost(10, x_data, y_data, 75)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.556182Z","iopub.execute_input":"2025-07-06T14:45:05.556411Z","iopub.status.idle":"2025-07-06T14:45:05.571081Z","shell.execute_reply.started":"2025-07-06T14:45:05.556388Z","shell.execute_reply":"2025-07-06T14:45:05.570282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(maes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.571900Z","iopub.execute_input":"2025-07-06T14:45:05.572595Z","iopub.status.idle":"2025-07-06T14:45:05.586033Z","shell.execute_reply.started":"2025-07-06T14:45:05.572574Z","shell.execute_reply":"2025-07-06T14:45:05.585307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(seeds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.586739Z","iopub.execute_input":"2025-07-06T14:45:05.586963Z","iopub.status.idle":"2025-07-06T14:45:05.601369Z","shell.execute_reply.started":"2025-07-06T14:45:05.586948Z","shell.execute_reply":"2025-07-06T14:45:05.600565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.602020Z","iopub.execute_input":"2025-07-06T14:45:05.602241Z","iopub.status.idle":"2025-07-06T14:45:05.616277Z","shell.execute_reply.started":"2025-07-06T14:45:05.602226Z","shell.execute_reply":"2025-07-06T14:45:05.615607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_to_sum_one(values):\n    total = sum(values)\n    if total == 0:\n        raise ValueError(\"Cannot normalize list with sum = 0.\")\n    return [x / total for x in values]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.617302Z","iopub.execute_input":"2025-07-06T14:45:05.617570Z","iopub.status.idle":"2025-07-06T14:45:05.631879Z","shell.execute_reply.started":"2025-07-06T14:45:05.617546Z","shell.execute_reply":"2025-07-06T14:45:05.631221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# weights = normalize_to_sum_one(weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.632494Z","iopub.execute_input":"2025-07-06T14:45:05.632685Z","iopub.status.idle":"2025-07-06T14:45:05.646967Z","shell.execute_reply.started":"2025-07-06T14:45:05.632670Z","shell.execute_reply":"2025-07-06T14:45:05.646332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_model_prediction_with_confidence(x_values, models, model_weights):\n    if len(models) != len(model_weights):\n        raise ValueError(\"Number of models and weights must be the same.\")\n    if not abs(sum(model_weights) - 1.0) < 1e-6:\n        raise ValueError(\"Model weights must sum to 1.\")\n\n    all_preds = [np.array(model.predict(x_values)) for model in models]  # shape: [num_models, num_samples]\n    all_preds = np.array(all_preds)  # shape: (n_models, n_samples)\n\n    # Weighted average prediction\n    weights = np.array(model_weights).reshape(-1, 1)  # shape: (n_models, 1)\n    weighted_pred = np.sum(weights * all_preds, axis=0)  # shape: (n_samples,)\n\n    # Weighted standard deviation (a proxy for uncertainty or confidence)\n    weighted_mean = weighted_pred\n    variance = np.sum(weights * (all_preds - weighted_mean) ** 2, axis=0)\n    std_dev = np.sqrt(variance)  # shape: (n_samples,)\n    confidence = 1 / (1 + std_dev) * 100 # confidence ∈ (0, 1], higher = more confident\n    confidence = np.clip(confidence, 420, 1000)\n\n    return weighted_pred.tolist(), confidence.tolist()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.648004Z","iopub.execute_input":"2025-07-06T14:45:05.648238Z","iopub.status.idle":"2025-07-06T14:45:05.663328Z","shell.execute_reply.started":"2025-07-06T14:45:05.648220Z","shell.execute_reply":"2025-07-06T14:45:05.662628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# x_train, x_val, y_train, y_val = train_test_split(x_data, y_data, test_size=0.2, random_state=17)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.663970Z","iopub.execute_input":"2025-07-06T14:45:05.664187Z","iopub.status.idle":"2025-07-06T14:45:05.682407Z","shell.execute_reply.started":"2025-07-06T14:45:05.664172Z","shell.execute_reply":"2025-07-06T14:45:05.681884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ada_boost_preds, conf = weighted_model_prediction_with_confidence(x_val, models, weights)\n# mae = mean_absolute_error(y_val, ada_boost_preds)\n# print(mae)\n\n# print(conf[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.683031Z","iopub.execute_input":"2025-07-06T14:45:05.683225Z","iopub.status.idle":"2025-07-06T14:45:05.697524Z","shell.execute_reply.started":"2025-07-06T14:45:05.683211Z","shell.execute_reply":"2025-07-06T14:45:05.696843Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"# test = pd.read_csv(base_path + 'test.csv')\n# sub_sample = pd.read_csv(base_path + 'sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.698296Z","iopub.execute_input":"2025-07-06T14:45:05.698504Z","iopub.status.idle":"2025-07-06T14:45:05.712927Z","shell.execute_reply.started":"2025-07-06T14:45:05.698480Z","shell.execute_reply":"2025-07-06T14:45:05.712365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(test.head())\n# print(sub_sample.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.713611Z","iopub.execute_input":"2025-07-06T14:45:05.713778Z","iopub.status.idle":"2025-07-06T14:45:05.728917Z","shell.execute_reply.started":"2025-07-06T14:45:05.713764Z","shell.execute_reply":"2025-07-06T14:45:05.728280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Extract Patient ID and Target Week from Patient_Week in df2\n# sub_sample[['Patient_ID', 'Target_Week']] = sub_sample['Patient_Week'].str.rsplit('_', n=1, expand=True)\n# sub_sample['Target_Week'] = sub_sample['Target_Week'].astype(int)\n\n# # Rename 'Patient' in df1 to match the new column name\n# test = test.rename(columns={'Patient': 'Patient_ID', 'Weeks': 'Reading_Week'})\n\n# # Merge the two DataFrames on Patient_ID (one-to-one mapping assumed)\n# merged = test.merge(sub_sample[['Patient_ID', 'Target_Week']], on='Patient_ID')\n\n# # Compute Weeks_diff\n# merged['Weeks_Diff'] = merged['Target_Week'] - merged['Reading_Week']\n\n# # Rename/Select columns for final format\n# final_df = merged[[\n#     'Patient_ID',\n#     'Reading_Week',\n#     'Target_Week',\n#     'Weeks_Diff',\n#     'FVC',          # This is Curr_FVC\n#     'Percent',\n#     'Age',\n#     'Sex',\n#     'SmokingStatus'\n# ]].rename(columns={'FVC': 'Curr_FVC'})\n\n# print(final_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.729842Z","iopub.execute_input":"2025-07-06T14:45:05.730116Z","iopub.status.idle":"2025-07-06T14:45:05.744179Z","shell.execute_reply.started":"2025-07-06T14:45:05.730080Z","shell.execute_reply":"2025-07-06T14:45:05.743667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Encode categorical variables\n# le_sex = LabelEncoder().fit(final_df['Sex'])\n# le_smoke = LabelEncoder().fit(final_df['SmokingStatus'])\n\n# final_df['Sex'] = le_sex.transform(final_df['Sex'])\n# final_df['SmokingStatus'] = le_smoke.transform(final_df['SmokingStatus'])\n\n# # Select features and target\n# features = ['Reading_Week', 'Target_Week', 'Weeks_Diff', 'Curr_FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus']\n\n# X = final_df[features]\n\n# # Normalize continuous features\n# scaler = StandardScaler()\n# X_scaled = scaler.fit_transform(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.745187Z","iopub.execute_input":"2025-07-06T14:45:05.745427Z","iopub.status.idle":"2025-07-06T14:45:05.765163Z","shell.execute_reply.started":"2025-07-06T14:45:05.745408Z","shell.execute_reply":"2025-07-06T14:45:05.764415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ada_boost_preds, conf = weighted_model_prediction_with_confidence(X_scaled, models, weights)\n# print(ada_boost_preds[:10])\n# print(conf[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.765919Z","iopub.execute_input":"2025-07-06T14:45:05.766166Z","iopub.status.idle":"2025-07-06T14:45:05.780703Z","shell.execute_reply.started":"2025-07-06T14:45:05.766146Z","shell.execute_reply":"2025-07-06T14:45:05.780108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(len(ada_boost_preds))\n# print(len(X_scaled))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.784251Z","iopub.execute_input":"2025-07-06T14:45:05.784527Z","iopub.status.idle":"2025-07-06T14:45:05.796357Z","shell.execute_reply.started":"2025-07-06T14:45:05.784508Z","shell.execute_reply":"2025-07-06T14:45:05.795884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Attach predictions and confidence to the DataFrame\n# final_df['FVC'] = ada_boost_preds\n# final_df['Confidence'] = conf\n\n# # Create the Patient_Week column\n# final_df['Patient_Week'] = final_df['Patient_ID'] + '_' + final_df['Target_Week'].astype(str)\n\n# # Select the final submission columns\n# submission = final_df[['Patient_Week', 'FVC', 'Confidence']].copy()\n\n# # Optional: round to reasonable precision\n# submission['FVC'] = submission['FVC'].round(1)\n# submission['Confidence'] = submission['Confidence'].round(1)\n\n# # Save or print\n# print(submission.head())\n# print(submission.shape)\n# submission.to_csv(\"submission.csv\", index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.797038Z","iopub.execute_input":"2025-07-06T14:45:05.797231Z","iopub.status.idle":"2025-07-06T14:45:05.813743Z","shell.execute_reply.started":"2025-07-06T14:45:05.797215Z","shell.execute_reply":"2025-07-06T14:45:05.813046Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Simple MLP","metadata":{}},{"cell_type":"code","source":"def compute_loss(y_pred, y_true, sigma):\n    f = torch.sqrt(torch.tensor(2.0, device=y_pred.device))\n    # sigma = torch.clamp(sigma, min=70.0)  # element-wise clamp\n\n    # Compute delta = min(1000, abs(y_pred - y_true))\n    delta = torch.abs(y_pred - y_true)\n    delta = torch.clamp(delta, max=1000.0)\n\n    # Compute loss\n    loss = f * delta / sigma + torch.log(f * sigma)\n\n    return loss.mean()  # return scalar loss\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.814556Z","iopub.execute_input":"2025-07-06T14:45:05.814806Z","iopub.status.idle":"2025-07-06T14:45:05.832076Z","shell.execute_reply.started":"2025-07-06T14:45:05.814786Z","shell.execute_reply":"2025-07-06T14:45:05.831414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ImprovedMLP(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.model = nn.Sequential(\n            nn.Linear(8, 64),\n            nn.ReLU(),\n            \n            nn.Linear(64, 64),\n            nn.ReLU(),\n            \n            nn.Linear(64, 32),\n            nn.ReLU(),\n            \n            nn.Linear(32, 2)\n        )\n\n    def forward(self, x):\n        return self.model(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.832885Z","iopub.execute_input":"2025-07-06T14:45:05.833134Z","iopub.status.idle":"2025-07-06T14:45:05.847366Z","shell.execute_reply.started":"2025-07-06T14:45:05.833115Z","shell.execute_reply":"2025-07-06T14:45:05.846629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SimpleMLP(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.model = nn.Sequential(\n            nn.Linear(8, 64),\n            nn.ReLU(),\n            nn.Linear(64, 32),\n            nn.ReLU(),\n            nn.Linear(32, 2)\n        )\n\n    def forward(self, x):\n        return self.model(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.848097Z","iopub.execute_input":"2025-07-06T14:45:05.848327Z","iopub.status.idle":"2025-07-06T14:45:05.862898Z","shell.execute_reply.started":"2025-07-06T14:45:05.848306Z","shell.execute_reply":"2025-07-06T14:45:05.862189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_MLP(model, device, optimizer, train_loader, val_loader, num_epochs=250):\n    best_model = model\n    min_loss = 1000\n    no_imprv = 0\n    \n    # Training loop\n    for epoch in range(num_epochs):  # number of epochs\n        model.train()\n        total_loss = 0\n        for xb, yb in train_loader:\n            xb, yb = xb.to(device), yb.to(device)\n    \n            pred = model(xb)              # shape: [batch_size, 2]\n            y_pred = pred[:, 0]           # shape: [batch_size]\n            sigma = pred[:, 1]            # shape: [batch_size]\n            \n            loss = compute_loss(y_pred, yb, sigma)\n    \n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n    \n            total_loss += loss.item()\n    \n        model.eval()\n        with torch.no_grad():\n            val_loss = 0\n            for xb, yb in val_loader:\n                xb, yb = xb.to(device), yb.to(device)\n                pred = model(xb)\n                y_pred, sigma = pred[:, 0], pred[:, 1]\n                loss = compute_loss(y_pred, yb, sigma)\n                val_loss += loss.item()\n\n            val_loss = val_loss / len(val_loader)\n            if val_loss - min_loss < 0:\n                min_loss = val_loss\n                best_model = model\n                print(f\"new best model with loss {val_loss}\")\n            else:\n                no_imprv += 1\n    \n        if no_imprv == 20:\n            print(\"Early stopping...\")\n            break\n    \n    return best_model, min_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.863642Z","iopub.execute_input":"2025-07-06T14:45:05.864262Z","iopub.status.idle":"2025-07-06T14:45:05.877550Z","shell.execute_reply.started":"2025-07-06T14:45:05.864237Z","shell.execute_reply":"2025-07-06T14:45:05.876832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = x_processed\ny = y_processed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.878328Z","iopub.execute_input":"2025-07-06T14:45:05.878540Z","iopub.status.idle":"2025-07-06T14:45:05.895112Z","shell.execute_reply.started":"2025-07-06T14:45:05.878525Z","shell.execute_reply":"2025-07-06T14:45:05.894492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ensemble_MLPs(n, X, Y):\n    models = []\n    weights = []\n    for _ in range(n):\n        x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.2, random_state=random.randint(1, 1000))\n        y_train = y_train.to_numpy()\n        y_val = y_val.to_numpy()\n\n        x_train_tensor = torch.tensor(x_train, dtype=torch.float32)\n        y_train_tensor = torch.tensor(y_train, dtype=torch.float32)\n        \n        x_val_tensor = torch.tensor(x_val, dtype=torch.float32)\n        y_val_tensor = torch.tensor(y_val, dtype=torch.float32)\n\n        train_dataset = TensorDataset(x_train_tensor, y_train_tensor)\n        val_dataset = TensorDataset(x_val_tensor, y_val_tensor)\n        \n        train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)\n        \n        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        model = SimpleMLP().to(device)\n        optimizer = optim.Adam(model.parameters(), lr=1e-3,  weight_decay=1e-5)\n        \n        mlp, loss = train_MLP(model=model,\n                               device=device,\n                               optimizer=optimizer,\n                               train_loader=train_loader,\n                               val_loader=val_loader)\n\n        if loss < 10:\n            models.append(mlp)\n            weights.append(math.log((20 - loss) / loss))\n\n    return models, weights\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.895784Z","iopub.execute_input":"2025-07-06T14:45:05.896001Z","iopub.status.idle":"2025-07-06T14:45:05.912078Z","shell.execute_reply.started":"2025-07-06T14:45:05.895986Z","shell.execute_reply":"2025-07-06T14:45:05.911346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.912729Z","iopub.execute_input":"2025-07-06T14:45:05.912985Z","iopub.status.idle":"2025-07-06T14:45:05.981944Z","shell.execute_reply.started":"2025-07-06T14:45:05.912968Z","shell.execute_reply":"2025-07-06T14:45:05.981122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ensemble_models, ensemble_weights = ensemble_MLPs(10, x, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:05.982838Z","iopub.execute_input":"2025-07-06T14:45:05.983124Z","iopub.status.idle":"2025-07-06T14:45:48.982152Z","shell.execute_reply.started":"2025-07-06T14:45:05.983103Z","shell.execute_reply":"2025-07-06T14:45:48.981174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(ensemble_models))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:48.983093Z","iopub.execute_input":"2025-07-06T14:45:48.983604Z","iopub.status.idle":"2025-07-06T14:45:48.987914Z","shell.execute_reply.started":"2025-07-06T14:45:48.983570Z","shell.execute_reply":"2025-07-06T14:45:48.987336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ensemble_weights = normalize_to_sum_one(ensemble_weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:48.988594Z","iopub.execute_input":"2025-07-06T14:45:48.988770Z","iopub.status.idle":"2025-07-06T14:45:49.006291Z","shell.execute_reply.started":"2025-07-06T14:45:48.988749Z","shell.execute_reply":"2025-07-06T14:45:49.005694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv(base_path + 'test.csv')\nsub_sample = pd.read_csv(base_path + 'sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:49.007261Z","iopub.execute_input":"2025-07-06T14:45:49.008052Z","iopub.status.idle":"2025-07-06T14:45:49.030223Z","shell.execute_reply.started":"2025-07-06T14:45:49.008027Z","shell.execute_reply":"2025-07-06T14:45:49.029580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract Patient ID and Target Week from Patient_Week in df2\nsub_sample[['Patient_ID', 'Target_Week']] = sub_sample['Patient_Week'].str.rsplit('_', n=1, expand=True)\nsub_sample['Target_Week'] = sub_sample['Target_Week'].astype(int)\n\n# Rename 'Patient' in df1 to match the new column name\ntest = test.rename(columns={'Patient': 'Patient_ID', 'Weeks': 'Reading_Week'})\n\n# Merge the two DataFrames on Patient_ID (one-to-one mapping assumed)\nmerged = test.merge(sub_sample[['Patient_ID', 'Target_Week']], on='Patient_ID')\n\n# Compute Weeks_diff\nmerged['Weeks_Diff'] = merged['Target_Week'] - merged['Reading_Week']\n\n# Rename/Select columns for final format\nfinal_df = merged[[\n    'Patient_ID',\n    'Reading_Week',\n    'Target_Week',\n    'Weeks_Diff',\n    'FVC',          # This is Curr_FVC\n    'Percent',\n    'Age',\n    'Sex',\n    'SmokingStatus'\n]].rename(columns={'FVC': 'Curr_FVC'})\n\nprint(final_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:49.031031Z","iopub.execute_input":"2025-07-06T14:45:49.031333Z","iopub.status.idle":"2025-07-06T14:45:49.050841Z","shell.execute_reply.started":"2025-07-06T14:45:49.031306Z","shell.execute_reply":"2025-07-06T14:45:49.050155Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Encode categorical variables\nle_sex = LabelEncoder().fit(final_df['Sex'])\nle_smoke = LabelEncoder().fit(final_df['SmokingStatus'])\n\nfinal_df['Sex'] = le_sex.transform(final_df['Sex'])\nfinal_df['SmokingStatus'] = le_smoke.transform(final_df['SmokingStatus'])\n\n# Select features and target\nfeatures = ['Reading_Week', 'Target_Week', 'Weeks_Diff', 'Curr_FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus']\n\nX = final_df[features]\n\n# Normalize continuous features\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:49.051588Z","iopub.execute_input":"2025-07-06T14:45:49.051854Z","iopub.status.idle":"2025-07-06T14:45:49.062143Z","shell.execute_reply.started":"2025-07-06T14:45:49.051832Z","shell.execute_reply":"2025-07-06T14:45:49.061588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_tensor = torch.tensor(X_scaled, dtype=torch.float32).to(device)  # shape: [12158, 8]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:49.062840Z","iopub.execute_input":"2025-07-06T14:45:49.063133Z","iopub.status.idle":"2025-07-06T14:45:49.075986Z","shell.execute_reply.started":"2025-07-06T14:45:49.063107Z","shell.execute_reply":"2025-07-06T14:45:49.075297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = []\nconfs = []\nfor model in ensemble_models:\n    model.to(device)\n    model.eval()\n    with torch.no_grad():\n        p = model(x_tensor)\n        p = p.cpu().detach().numpy()\n        preds.append(p[:, 0])\n        confs.append(p[:, 1])\n\n# preds.append(ada_boost_preds)\n# confs.append(conf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:49.076871Z","iopub.execute_input":"2025-07-06T14:45:49.077112Z","iopub.status.idle":"2025-07-06T14:45:49.092548Z","shell.execute_reply.started":"2025-07-06T14:45:49.077090Z","shell.execute_reply":"2025-07-06T14:45:49.092072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# mean_preds = np.mean(preds, axis=0)\n# mean_confs = np.mean(confs, axis=0)\n\nweights = np.array(ensemble_weights).reshape(-1, 1)  # shape: (n_models, 1)\nweighted_pred = np.sum(weights * preds, axis=0)  # shape: (n_samples,)\nweighted_conf = np.sum(weights * confs, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:45:49.093167Z","iopub.execute_input":"2025-07-06T14:45:49.093423Z","iopub.status.idle":"2025-07-06T14:45:49.107088Z","shell.execute_reply.started":"2025-07-06T14:45:49.093401Z","shell.execute_reply":"2025-07-06T14:45:49.106343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(weighted_pred[:10])\nprint(weighted_conf[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:46:25.036275Z","iopub.execute_input":"2025-07-06T14:46:25.036546Z","iopub.status.idle":"2025-07-06T14:46:25.041538Z","shell.execute_reply.started":"2025-07-06T14:46:25.036525Z","shell.execute_reply":"2025-07-06T14:46:25.040823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Attach predictions and confidence to the DataFrame\nfinal_df['FVC'] = weighted_pred\nfinal_df['Confidence'] = weighted_conf\n\n# Create the Patient_Week column\nfinal_df['Patient_Week'] = final_df['Patient_ID'] + '_' + final_df['Target_Week'].astype(str)\n\n# Select the final submission columns\nsubmission = final_df[['Patient_Week', 'FVC', 'Confidence']].copy()\n\n# Optional: round to reasonable precision\nsubmission['FVC'] = submission['FVC'].round(1)\nsubmission['Confidence'] = submission['Confidence'].round(1)\n\n# Save or print\nprint(submission.head())\nprint(submission.shape)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-06T14:46:30.530034Z","iopub.execute_input":"2025-07-06T14:46:30.530517Z","iopub.status.idle":"2025-07-06T14:46:30.549125Z","shell.execute_reply.started":"2025-07-06T14:46:30.530493Z","shell.execute_reply":"2025-07-06T14:46:30.548460Z"}},"outputs":[],"execution_count":null}]}