{"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":1555593,"sourceType":"datasetVersion","datasetId":850380}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport re\nimport torch\nfrom bs4 import BeautifulSoup\nfrom datasets import Dataset as HFDataset, DatasetDict\nfrom transformers import DistilBertTokenizer, DistilBertForSequenceClassification\nfrom transformers import RobertaTokenizer, RobertaForSequenceClassification\nfrom transformers import AlbertTokenizer, AlbertForSequenceClassification\nfrom transformers import ElectraTokenizer, ElectraForSequenceClassification\nfrom transformers import XLNetTokenizer, XLNetForSequenceClassification\nfrom transformers import BertTokenizer, BertForSequenceClassification\nfrom torch.utils.data import DataLoader\nfrom transformers import get_scheduler\nfrom torch.optim import AdamW\nfrom tqdm import tqdm\nimport spacy\nimport warnings\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\nimport joblib","metadata":{"id":"4ri7NDocWwGL","execution":{"iopub.status.busy":"2024-06-08T06:35:52.623234Z","iopub.execute_input":"2024-06-08T06:35:52.623564Z","iopub.status.idle":"2024-06-08T06:36:06.173037Z","shell.execute_reply.started":"2024-06-08T06:35:52.623537Z","shell.execute_reply":"2024-06-08T06:36:06.172214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize Spacy and load stopwords\nnlp = spacy.load(\"en_core_web_sm\")\nstopwords = nlp.Defaults.stop_words\n\n# Ignore BeautifulSoup warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module='bs4')\n\n# Preprocessing Class\nclass Preprocessing:\n    def __get_body_tag_text(self, text):\n        soup = BeautifulSoup(text, features='xml')\n        return soup.get_text()\n\n    def __lower_and_punc_removal(self, text):\n        text = text.lower()\n        text = re.sub(r'[^(a-zA-Z)\\s]', '', text)\n        return text\n\n    def __init__(self, df):\n        self.df = df\n        self.df['Body_Between_Tags'] = self.df['Body'].apply(self.__get_body_tag_text)\n        self.df['Final_clean'] = self.df['Body_Between_Tags'].apply(self.__lower_and_punc_removal)","metadata":{"id":"BnAdcs7yW691","execution":{"iopub.status.busy":"2024-06-08T06:36:06.174498Z","iopub.execute_input":"2024-06-08T06:36:06.175033Z","iopub.status.idle":"2024-06-08T06:36:07.416123Z","shell.execute_reply.started":"2024-06-08T06:36:06.175008Z","shell.execute_reply":"2024-06-08T06:36:07.415300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load data\ndf_train = pd.read_csv('/kaggle/input/60k-stack-overflow-questions-with-quality-rate/train.csv')\ndf_valid = pd.read_csv('/kaggle/input/60k-stack-overflow-questions-with-quality-rate/valid.csv')\n\n# Map labels to integers\ndf_train['Y'] = df_train['Y'].map({'LQ_CLOSE':0, 'LQ_EDIT': 1, 'HQ':2})\ndf_valid['Y'] = df_valid['Y'].map({'LQ_CLOSE':0, 'LQ_EDIT': 1, 'HQ':2})","metadata":{"id":"mPTUTqnRW669","execution":{"iopub.status.busy":"2024-06-08T06:36:07.417221Z","iopub.execute_input":"2024-06-08T06:36:07.417522Z","iopub.status.idle":"2024-06-08T06:36:09.072849Z","shell.execute_reply.started":"2024-06-08T06:36:07.417498Z","shell.execute_reply":"2024-06-08T06:36:09.071633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess data\nPreprocessedObject_train = Preprocessing(df=df_train)\nPreprocessedObject_valid = Preprocessing(df=df_valid)\n\ndf_train_final = PreprocessedObject_train.df.dropna().reset_index(drop=True)\ndf_valid_final = PreprocessedObject_valid.df.dropna().reset_index(drop=True)\n\ndf_train_final = df_train_final[['Final_clean', 'Y']]\ndf_valid_final = df_valid_final[['Final_clean', 'Y']]","metadata":{"id":"vHi6h2HKW64m","execution":{"iopub.status.busy":"2024-06-08T06:36:09.075369Z","iopub.execute_input":"2024-06-08T06:36:09.075706Z","iopub.status.idle":"2024-06-08T06:36:17.888657Z","shell.execute_reply.started":"2024-06-08T06:36:09.075680Z","shell.execute_reply":"2024-06-08T06:36:17.887604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create datasets\ndatasets_train_test = DatasetDict({\n    \"train\": HFDataset.from_pandas(df_train_final),\n    \"test\": HFDataset.from_pandas(df_valid_final)\n})","metadata":{"id":"Q9OY-VRLW6P1","execution":{"iopub.status.busy":"2024-06-08T06:36:17.889955Z","iopub.execute_input":"2024-06-08T06:36:17.890302Z","iopub.status.idle":"2024-06-08T06:36:17.974381Z","shell.execute_reply.started":"2024-06-08T06:36:17.890276Z","shell.execute_reply":"2024-06-08T06:36:17.973401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define tokenizers and models\ntokenizers_and_models = [\n    (DistilBertTokenizer.from_pretrained('distilbert-base-uncased'), DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=3)),\n    (RobertaTokenizer.from_pretrained('roberta-base'), RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=3)),\n    (AlbertTokenizer.from_pretrained('albert-base-v2'), AlbertForSequenceClassification.from_pretrained('albert-base-v2', num_labels=3)),\n    (ElectraTokenizer.from_pretrained('google/electra-base-discriminator'), ElectraForSequenceClassification.from_pretrained('google/electra-base-discriminator', num_labels=3)),\n    (XLNetTokenizer.from_pretrained('xlnet-base-cased'), XLNetForSequenceClassification.from_pretrained('xlnet-base-cased', num_labels=3)),\n    (BertTokenizer.from_pretrained('bert-base-uncased'), BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=3))\n]\n\n# Helper function to preprocess and tokenize\ndef tokenize_function(examples, tokenizer):\n    return tokenizer(examples[\"Final_clean\"], padding=\"max_length\", truncation=True, max_length=128)\n\ndef get_dataloader(dataset, tokenizer, batch_size=8):\n    tokenized_dataset = dataset.map(lambda x: tokenize_function(x, tokenizer), batched=True)\n    tokenized_dataset = tokenized_dataset.remove_columns(['Final_clean'])\n    tokenized_dataset = tokenized_dataset.rename_column(\"Y\", \"labels\")\n    tokenized_dataset.set_format(\"torch\")\n    dataloader = DataLoader(tokenized_dataset, shuffle=True, batch_size=batch_size)\n    return dataloader\n\ntrain_dataloaders = [get_dataloader(datasets_train_test[\"train\"], tokenizer) for tokenizer, _ in tokenizers_and_models]\nvalid_dataloaders = [get_dataloader(datasets_train_test[\"test\"], tokenizer) for tokenizer, _ in tokenizers_and_models]","metadata":{"id":"5UweL9QVW6MV","execution":{"iopub.status.busy":"2024-06-08T06:36:17.975777Z","iopub.execute_input":"2024-06-08T06:36:17.976396Z","iopub.status.idle":"2024-06-08T06:40:36.631264Z","shell.execute_reply.started":"2024-06-08T06:36:17.976361Z","shell.execute_reply":"2024-06-08T06:40:36.630459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fine-tune each model\ndef train_model(model, dataloader, epochs=1, device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")):\n    optimizer = AdamW(model.parameters(), lr=5e-5)\n    num_training_steps = epochs * len(dataloader)\n    lr_scheduler = get_scheduler(\n        name=\"linear\", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps\n    )\n\n    model.to(device)\n    progress_bar = tqdm(range(num_training_steps))\n\n    model.train()\n    for epoch in range(epochs):\n        for batch in dataloader:\n            batch = {k: v.to(device) for k, v in batch.items()}\n            outputs = model(**batch)\n            loss = outputs.loss\n            loss.backward()\n\n            optimizer.step()\n            lr_scheduler.step()\n            optimizer.zero_grad()\n            progress_bar.update(1)\n    return model","metadata":{"id":"OjcDHBLvW6Jd","execution":{"iopub.status.busy":"2024-06-08T06:40:36.632619Z","iopub.execute_input":"2024-06-08T06:40:36.632997Z","iopub.status.idle":"2024-06-08T06:40:36.641475Z","shell.execute_reply.started":"2024-06-08T06:40:36.632963Z","shell.execute_reply":"2024-06-08T06:40:36.640627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trained_models = [train_model(model, dataloader) for (_, model), dataloader in zip(tokenizers_and_models, train_dataloaders)]","metadata":{"id":"0He-Eir3W6GN","execution":{"iopub.status.busy":"2024-06-08T06:40:36.642467Z","iopub.execute_input":"2024-06-08T06:40:36.642716Z","iopub.status.idle":"2024-06-08T08:21:53.460576Z","shell.execute_reply.started":"2024-06-08T06:40:36.642693Z","shell.execute_reply":"2024-06-08T08:21:53.459445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save trained models\nfor i, (_, model) in enumerate(zip(tokenizers_and_models, trained_models)):\n    torch.save(model.state_dict(), f'model_{i}.pt')","metadata":{"id":"OopNS6XhW6Dt","execution":{"iopub.status.busy":"2024-06-08T08:21:53.462095Z","iopub.execute_input":"2024-06-08T08:21:53.462466Z","iopub.status.idle":"2024-06-08T08:21:56.628034Z","shell.execute_reply.started":"2024-06-08T08:21:53.462433Z","shell.execute_reply":"2024-06-08T08:21:56.627202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate predictions for stacking\ndef get_predictions(model, dataloader, device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")):\n    model.to(device)\n    model.eval()\n    predictions = []\n    with torch.no_grad():\n        for batch in dataloader:\n            batch = {k: v.to(device) for k, v in batch.items()}\n            outputs = model(**batch)\n            logits = outputs.logits\n            preds = torch.argmax(logits, dim=-1)\n            predictions.extend(preds.cpu().numpy())\n    return np.array(predictions)","metadata":{"id":"TfcM5oraW6A-","execution":{"iopub.status.busy":"2024-06-08T08:21:56.631389Z","iopub.execute_input":"2024-06-08T08:21:56.631928Z","iopub.status.idle":"2024-06-08T08:21:56.638276Z","shell.execute_reply.started":"2024-06-08T08:21:56.631899Z","shell.execute_reply":"2024-06-08T08:21:56.637376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_predictions = np.vstack([get_predictions(model, dataloader) for model, dataloader in zip(trained_models, train_dataloaders)]).T\nvalid_predictions = np.vstack([get_predictions(model, dataloader) for model, dataloader in zip(trained_models, valid_dataloaders)]).T","metadata":{"id":"gAXsgxtSW5-d","execution":{"iopub.status.busy":"2024-06-08T08:21:56.639485Z","iopub.execute_input":"2024-06-08T08:21:56.640077Z","iopub.status.idle":"2024-06-08T09:04:09.306225Z","shell.execute_reply.started":"2024-06-08T08:21:56.640045Z","shell.execute_reply":"2024-06-08T09:04:09.305359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train meta-learner\nmeta_learner = LogisticRegression()\nmeta_learner.fit(train_predictions, df_train_final['Y'])\n\n# Save meta-learner\njoblib.dump(meta_learner, 'logistic_regression_meta_learner.joblib')\n\n# Evaluate meta-learner\nvalid_preds = meta_learner.predict(valid_predictions)\naccuracy = accuracy_score(df_valid_final['Y'], valid_preds)\nprint(\"Stacking Model Accuracy:\", accuracy)","metadata":{"id":"eObFHrloW57W","execution":{"iopub.status.busy":"2024-06-08T09:04:09.307500Z","iopub.execute_input":"2024-06-08T09:04:09.307854Z","iopub.status.idle":"2024-06-08T09:04:09.479599Z","shell.execute_reply.started":"2024-06-08T09:04:09.307822Z","shell.execute_reply":"2024-06-08T09:04:09.478385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define parameter grid for KNN\nparam_grid = {'n_neighbors': np.arange(1, 51)}\n\n# Initialize KNN and GridSearchCV\nknn = KNeighborsClassifier()\ngrid_search = GridSearchCV(knn, param_grid, cv=5, scoring='accuracy')\ngrid_search.fit(train_predictions, df_train_final['Y'])\n\n# Best KNN model\nbest_knn = grid_search.best_estimator_\n\n# Save KNN meta-learner\njoblib.dump(best_knn, 'knn_meta_learner.joblib')\n\n# Evaluate KNN meta-learner\nvalid_preds = best_knn.predict(valid_predictions)\naccuracy = accuracy_score(df_valid_final['Y'], valid_preds)\nprint(\"Stacking Model Accuracy with KNN Meta-Learner:\", accuracy)","metadata":{"id":"12EdB0TcW52G","execution":{"iopub.status.busy":"2024-06-08T09:04:09.481408Z","iopub.execute_input":"2024-06-08T09:04:09.481939Z","iopub.status.idle":"2024-06-08T09:08:21.216141Z","shell.execute_reply.started":"2024-06-08T09:04:09.481890Z","shell.execute_reply":"2024-06-08T09:08:21.215200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"aE3c2vkTW5y-"},"execution_count":null,"outputs":[]}]}