{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":12088794,"sourceType":"datasetVersion","datasetId":7609988}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Multiple linear regression analysis for prediction of optimal parameters","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nfrom itertools import product\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import TensorDataset, DataLoader\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T12:13:43.802999Z","iopub.execute_input":"2025-06-07T12:13:43.803458Z","iopub.status.idle":"2025-06-07T12:13:43.944399Z","shell.execute_reply.started":"2025-06-07T12:13:43.803423Z","shell.execute_reply":"2025-06-07T12:13:43.943437Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data initialisation","metadata":{}},{"cell_type":"markdown","source":"This is all the data gathered on the inference notebook with the unknown model. Ofcourse when using this notebook for prediction purposes there was less data available. ","metadata":{}},{"cell_type":"code","source":"# Data initialisation\ndata = [\n    [0.836, 1536, 64, 256, 16000],\n    [0.835, 2048, 64, 256, 16000],\n    [0.835, 2048, 128, 256, 16000],\n    [0.835, 1792, 64, 256, 16000],\n    [0.834, 1034, 64, 136, 16000],\n    [0.831, 2048, 128, 200, 16000],\n    [0.829, 2048, 512, 512, 16000],\n    [0.828, 1536, 64, 512, 16000],\n    [0.824, 1280, 64, 256, 16000],\n    [0.814, 1152, 64, 136, 16000],\n    [0.814, 1024, 64, 256, 16000],\n    [0.810, 1034, 64, 180, 16000],\n    [0.810, 1034, 128, 130, 16000],\n    [0.810, 1024, 64, 144, 16000],\n    [0.806, 1024, 64, 120, 16000],\n    [0.803, 1536, 64, 256, 10000],\n    [0.795, 4096, 64, 448, 16000],\n    [0.773, 512, 128, 256, 16000],\n    [0.766, 512, 64, 256, 16000],\n    [0.764, 512, 128, 352, 16000],\n    [0.754, 1536, 128, 64, 16000],\n    [0.754, 1536, 64, 64, 16000],\n    [0.742, 1024, 256, 64, 12000],\n    [0.736, 512, 128, 64, 16000],\n    [0.734, 512, 64, 64, 16000],\n    [0.828, 1536, 128, 512, 16000],\n    [0.822, 1536, 64, 256, 14000],\n    [0.824, 1536, 256, 256, 14000],\n    [0.822, 2304, 64, 256, 14000],\n    [0.769, 512, 1024, 104, 16000],\n\n]\n\ndf = pd.DataFrame(data, columns=['LB', 'N_FFT', 'HOP_LENGTH', 'N_MELS', 'FMAX'])\n\n# Preprocess data\nX = df.drop('LB', axis=1).values\ny = df['LB'].values\n\n# Scale features\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)\n\n# Train/test split\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T11:48:29.067246Z","iopub.execute_input":"2025-06-07T11:48:29.067779Z","iopub.status.idle":"2025-06-07T11:48:29.081996Z","shell.execute_reply.started":"2025-06-07T11:48:29.067753Z","shell.execute_reply":"2025-06-07T11:48:29.081021Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Neural Network","metadata":{}},{"cell_type":"markdown","source":"We first tried a Neural Network as this approach is able to capture the non-linearities in the data. However, training a neural network with this little data means that it is prone to overfitting.","metadata":{}},{"cell_type":"code","source":"# Convert to tensors\nX_train_tensor = torch.tensor(X_train, dtype=torch.float32)\ny_train_tensor = torch.tensor(y_train, dtype=torch.float32).view(-1, 1)\nX_test_tensor = torch.tensor(X_test, dtype=torch.float32)\ny_test_tensor = torch.tensor(y_test, dtype=torch.float32).view(-1, 1)\n\n# Load data\ntrain_ds = TensorDataset(X_train_tensor, y_train_tensor)\ntrain_dl = DataLoader(train_ds, batch_size=4, shuffle=True)\n\n# Define Neural Network \nclass LBRegressor(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(4, 16),\n            nn.ReLU(),\n            nn.Linear(16, 8),\n            nn.ReLU(),\n            nn.Linear(8, 1)\n        )\n\n    def forward(self, x):\n        return self.net(x)\n\nmodel = LBRegressor()\n\n# Define loss function and optimizer\nloss_fn = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.01)\n\n# Training Loop\nepochs = 25\nlosses = []\n\nfor epoch in range(epochs):\n    model.train()\n    for xb, yb in train_dl:\n        pred = model(xb)\n        loss = loss_fn(pred, yb)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    losses.append(loss.item())\n\n# Evaluation\nmodel.eval()\nwith torch.no_grad():\n    preds = model(X_test_tensor).squeeze()\n    mse = ((preds - y_test_tensor.squeeze()) ** 2).mean().item()\n    print(f\"Test MSE: {mse:.5f}\")\n\n# Plotting the loss\nplt.plot(losses)\nplt.title(\"Training Loss\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"MSE Loss\")\nplt.grid(True)\nplt.show()\n\n# Compare true value and predicted value \nfor i in range(len(y_test)):\n    print(f\"Actual: {y_test[i]:.3f}, Predicted: {preds[i].item():.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T11:51:47.604621Z","iopub.execute_input":"2025-06-07T11:51:47.604924Z","iopub.status.idle":"2025-06-07T11:51:52.126022Z","shell.execute_reply.started":"2025-06-07T11:51:47.604903Z","shell.execute_reply":"2025-06-07T11:51:52.124813Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Predicting optimal parameters","metadata":{}},{"cell_type":"code","source":"# Search space \nsearch_space = {\n    'N_FFT': np.arange(512, 4097, 128),       \n    'HOP_LENGTH': np.arange(32, 1025, 32),\n    'N_MELS': np.arange(64, 513, 8),\n    'FMAX': [12000, 14000, 16000]          \n}\n\n# Generate parameter combinations \nN_FFT_options = [512, 1024, 1152, 1536, 2048]\nHOP_OPTIONS = [64, 128, 256, 512]\nMELS_OPTIONS = [64, 128, 136, 180, 200, 256]\nFMAX_OPTIONS = [12000, 16000]\n\nsearch_space = list(product(N_FFT_options, HOP_OPTIONS, MELS_OPTIONS, FMAX_OPTIONS))\n\n# Scale the parameter combinations\nrandom_inputs = np.array(search_space)\nrandom_inputs_scaled = scaler.transform(random_inputs)\n\n# Predict scores for parameter combinations\nmodel.eval()\nwith torch.no_grad():\n    inputs_tensor = torch.tensor(random_inputs_scaled, dtype=torch.float32)\n    preds = model(inputs_tensor).squeeze().numpy()\n\n# Sort Results \nresults_df = pd.DataFrame(random_inputs, columns=['N_FFT', 'HOP_LENGTH', 'N_MELS', 'FMAX'])\nresults_df['Predicted_LB'] = preds\nresult_1 = results_df.sort_values(by='Predicted_LB', ascending=False).reset_index(drop=True)\n\nprint(\"\\nTop 10 Predicted Configurations NN:\")\nprint(result_1.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T12:17:58.191237Z","iopub.execute_input":"2025-06-07T12:17:58.191593Z","iopub.status.idle":"2025-06-07T12:17:58.207538Z","shell.execute_reply.started":"2025-06-07T12:17:58.191563Z","shell.execute_reply":"2025-06-07T12:17:58.206471Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Linear regression","metadata":{}},{"cell_type":"code","source":"# Make Linear Regression model\nreg = LinearRegression()\nreg.fit(X_train, y_train)\n\n# Evaluation\nr2 = reg.score(X_test, y_test)\nprint(f\"\\nR² on test data: {r2:.4f}\")\nprint(f\"Coefficients: {reg.coef_}\")\nprint(f\"Intercept: {reg.intercept_}\")\n\n# Generate parameter combinations \nN_FFT_options = [512, 1024, 1152, 1536, 2048]\nHOP_OPTIONS = [64, 128, 256, 512]\nMELS_OPTIONS = [64, 128, 136, 180, 200, 256]\nFMAX_OPTIONS = [12000, 16000]\n\nsearch_space = list(product(N_FFT_options, HOP_OPTIONS, MELS_OPTIONS, FMAX_OPTIONS))\n\nresults = []\n\n# Predict scores for parameter combinations\nfor combo in search_space:\n    X_candidate = np.array(combo).reshape(1, -1)\n    X_scaled = scaler.transform(X_candidate)\n    pred_lb = reg.predict(X_scaled)[0]\n    results.append(round(pred_lb, 5))\n\n# Sort Results \nresults_df = pd.DataFrame(search_space, columns=['N_FFT', 'HOP_LENGTH', 'N_MELS', 'FMAX'])\nresults_df['Predicted_LB'] = results\nresult_LR = results_df.sort_values(by='Predicted_LB', ascending=False).reset_index(drop=True)\n\nprint(\"\\nTop 10 Predicted Configurations NN:\")\nprint(result_LR.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T12:25:41.323904Z","iopub.execute_input":"2025-06-07T12:25:41.324190Z","iopub.status.idle":"2025-06-07T12:25:41.392029Z","shell.execute_reply.started":"2025-06-07T12:25:41.324172Z","shell.execute_reply":"2025-06-07T12:25:41.391040Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Multivariate Multiple Linear Regression","metadata":{}},{"cell_type":"markdown","source":"Multivariate Multiple Linear Regression is used for getting the optimal parameter set for the training notebook that uses ResNet. In this notebook, the Mel-spectogram parameters and multiple scores (training AUROC, validation AUROC, training loss, validation loss) are kept track of.  ","metadata":{}},{"cell_type":"markdown","source":"### Data preparation","metadata":{}},{"cell_type":"code","source":"# This data is from the original notebook\ndata = [\n    [1, 0.77, 0.93, 3.6371, 0.6873, 1024, 512, 128],\n    [2, 0.93, 0.95, 2.4111, 0.5505, 1024, 512, 128],\n    [3, 0.96, 0.95, 1.8917, 0.5005, 1024, 512, 128],\n    [4, 0.98, 0.97, 1.5317, 0.4349, 1024, 512, 128],\n]\ndf_original_data = pd.DataFrame(data, columns = ['epoch', 'auc', 'auc_val', 'loss', 'loss_val', 'fft', 'hop', 'mels'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T12:43:53.712168Z","iopub.execute_input":"2025-06-07T12:43:53.712515Z","iopub.status.idle":"2025-06-07T12:43:53.720319Z","shell.execute_reply.started":"2025-06-07T12:43:53.712491Z","shell.execute_reply":"2025-06-07T12:43:53.719014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the .csv file\ndf = pd.read_csv('/kaggle/input/training-results-resnet/results_training.csv')\n\n# Extract values from modelnames\ndf['fft'] = df['name'].apply(lambda x: ''.join(c for c in x.split(\"_\")[2] if c.isdigit()))\ndf['hop'] = df['name'].apply(lambda x: ''.join(c for c in x.split(\"_\")[3] if c.isdigit()))\ndf['mels'] = df['name'].apply(lambda x: ''.join(c for c in x.split(\"_\")[4] if c.isdigit()))\ndf = df.drop(['name'], axis=1)\ndf = pd.concat([df, df_original_data]).reset_index().drop(['index'], axis=1)\n\n# Extract data for each epoch\nepoch1 = df[df['epoch']==1].drop(['epoch'], axis=1)\nepoch2 = df[df['epoch']==2].drop(['epoch'], axis=1)\nepoch3 = df[df['epoch']==3].drop(['epoch'], axis=1)\nepoch4 = df[df['epoch']==4].drop(['epoch'], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T12:43:54.165019Z","iopub.execute_input":"2025-06-07T12:43:54.165325Z","iopub.status.idle":"2025-06-07T12:43:54.184094Z","shell.execute_reply.started":"2025-06-07T12:43:54.165301Z","shell.execute_reply":"2025-06-07T12:43:54.182943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"epoch_list = [epoch1, epoch2, epoch3, epoch4]\n\nfor j in range(4):\n    print(f\"\\n{'='*20} Analyzing Epoch {j} {'='*20}\")\n\n    ys = ['auc', 'auc_val', 'loss', 'loss_val']\n    \n    # Multivariate targets\n    X = epoch_list[j].drop(columns=ys).values\n    Y = epoch_list[j][ys].values  # Y is now a matrix with 4 columns (multivariate)\n\n    # Standardize features\n    scaler = StandardScaler()\n    X_scaled = scaler.fit_transform(X)\n\n    # Train-test split\n    X_train, X_test, Y_train, Y_test = train_test_split(X_scaled, Y, test_size=0.2, random_state=42)\n\n    # Multivariate Linear Regression\n    reg = LinearRegression()\n    reg.fit(X_train, Y_train)\n\n    # Evaluate performance for each target\n    R2_scores = reg.score(X_test, Y_test)  # This is averaged R² across outputs\n    print(f\"Average R² on test data: {R2_scores:.4f}\")\n    print(f\"Coefficients (per output):\\n{reg.coef_}\")\n    print(f\"Intercepts (per output): {reg.intercept_}\")\n\n    # Parameter search space\n    N_FFT_options = [512, 768, 1024, 1152, 1280, 1536, 1792, 2048]\n    HOP_OPTIONS = [64, 86, 128, 192, 256, 512]\n    MELS_OPTIONS = [64, 128, 136, 180, 200, 256]\n\n    search_space = list(product(N_FFT_options, HOP_OPTIONS, MELS_OPTIONS))\n\n    predictions = []\n    for combo in search_space:\n        X_candidate = np.array(combo).reshape(1, -1)\n        X_scaled = scaler.transform(X_candidate)\n        preds = reg.predict(X_scaled)[0]\n        predictions.append((*combo, *preds))  # fft, hop, mels, auc, auc_val, loss, loss_val\n\n    results_df = pd.DataFrame(\n        predictions,\n        columns=['fft', 'hop', 'mels', 'auc', 'auc_val', 'loss', 'loss_val']\n    )\n\n    # Display sorted results for each target\n    for target in ys:\n        ascending = True if 'loss' in target else False\n        sorted_df = results_df.sort_values(by=target, ascending=ascending)\n        print(f\"\\n--- Top results sorted by '{target}' ---\")\n        display(sorted_df.reset_index(drop=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T12:45:48.278943Z","iopub.execute_input":"2025-06-07T12:45:48.279340Z","iopub.status.idle":"2025-06-07T12:45:48.640879Z","shell.execute_reply.started":"2025-06-07T12:45:48.279228Z","shell.execute_reply":"2025-06-07T12:45:48.639919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}