{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import the libraries\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport nltk","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:36.109683Z","iopub.execute_input":"2022-07-06T16:29:36.110325Z","iopub.status.idle":"2022-07-06T16:29:36.617701Z","shell.execute_reply.started":"2022-07-06T16:29:36.110287Z","shell.execute_reply":"2022-07-06T16:29:36.616841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load dat and check first few rows\ntrain_data = pd.read_csv(\"../input/feedback-prize-2021/train.csv\")\n\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:36.619737Z","iopub.execute_input":"2022-07-06T16:29:36.620026Z","iopub.status.idle":"2022-07-06T16:29:38.165120Z","shell.execute_reply.started":"2022-07-06T16:29:36.619989Z","shell.execute_reply":"2022-07-06T16:29:38.164346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check datatypes and null values etc\ntrain_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:38.167774Z","iopub.execute_input":"2022-07-06T16:29:38.167972Z","iopub.status.idle":"2022-07-06T16:29:38.270303Z","shell.execute_reply.started":"2022-07-06T16:29:38.167948Z","shell.execute_reply":"2022-07-06T16:29:38.269500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"'''lets plot the value count of all the classes. \nThis help us to understand if there is any class imbalance\nand total number of classes''' \n\nplt.figure(figsize = (12,5))\nsns.histplot(train_data['discourse_type'])","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:38.272321Z","iopub.execute_input":"2022-07-06T16:29:38.273228Z","iopub.status.idle":"2022-07-06T16:29:38.781404Z","shell.execute_reply.started":"2022-07-06T16:29:38.273186Z","shell.execute_reply":"2022-07-06T16:29:38.780596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''The number of count of all the labels are very different, therefore we can say that the dataset in higly imbalanced.\nLet us find the number of counts of each label\nI have defined a helper function to do that'''\n\ndef count_target(target_list):\n    target_dict = {}\n    for x in target_list:\n        count = len(train_data[train_data['discourse_type'] == x])\n        dict_t = dict({x:count})\n        target_dict.update(dict_t)\n    return target_dict\n\ntarget_list = ['Lead', 'Position', 'Evidence','Claim','Concluding Statement','Counterclaim','Rebuttal']\ncount_target(target_list)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:38.782882Z","iopub.execute_input":"2022-07-06T16:29:38.783370Z","iopub.status.idle":"2022-07-06T16:29:38.965299Z","shell.execute_reply.started":"2022-07-06T16:29:38.783329Z","shell.execute_reply":"2022-07-06T16:29:38.964558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data[['id','discourse_text','discourse_type','predictionstring']]\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:38.966692Z","iopub.execute_input":"2022-07-06T16:29:38.967157Z","iopub.status.idle":"2022-07-06T16:29:38.988783Z","shell.execute_reply.started":"2022-07-06T16:29:38.967117Z","shell.execute_reply":"2022-07-06T16:29:38.987962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. From the table we can see that it is a classification problem. based on the text context, model has to classify among various classes 'discourse _type'.\n2. We need to build a classifier which classify amoung various classes involved.\n3. Before that we need to do pre processing where we change all text to lower case and remove the punctuations.\n4. We want to proceed initially without removing the stopwords as they may paly important role in identifying the context of sentences.\n5. We are going to use tfidf vectorizer to convert the text document into vector.\n6. Steps to be involved.\n    1. remove punctuations and change to lower case\n    2. tokenize the document\n    3. find the tfidf vectors\n    4. split dataset into train and validation\n    5. train the model on train dataset to be able to classify among various calsses using few classification algorithm.\n    6. make prediction on validation dataset and check accuracy of model.\n    7. preprocess test dataset\n    8. make prediction on test dataset.\n    \n    ","metadata":{}},{"cell_type":"code","source":"import re\nimport string\nfrom nltk import sent_tokenize, word_tokenize\nfrom nltk.corpus import stopwords","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:38.990367Z","iopub.execute_input":"2022-07-06T16:29:38.990627Z","iopub.status.idle":"2022-07-06T16:29:38.994808Z","shell.execute_reply.started":"2022-07-06T16:29:38.990593Z","shell.execute_reply":"2022-07-06T16:29:38.994132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let define a helper function to remove the punctuation and change text to lower case\n# remove stop words from the string\ndef cleanup_text(text):\n    words = re.sub(pattern = '[^a-zA-Z]',repl = ' ',string = text)\n    words = words.lower()\n    #words = [word for word in words.split() if word not in stopwords.words('english')]\n    #final_sent = ' '.join(words)\n    return words   # return final_sent |if stopword removal is required","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:38.996252Z","iopub.execute_input":"2022-07-06T16:29:38.996745Z","iopub.status.idle":"2022-07-06T16:29:39.007542Z","shell.execute_reply.started":"2022-07-06T16:29:38.996705Z","shell.execute_reply":"2022-07-06T16:29:39.003126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_processed = train_data['discourse_text'].apply(cleanup_text)\ntext_processed","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:39.010361Z","iopub.execute_input":"2022-07-06T16:29:39.010628Z","iopub.status.idle":"2022-07-06T16:29:42.416763Z","shell.execute_reply.started":"2022-07-06T16:29:39.010594Z","shell.execute_reply":"2022-07-06T16:29:42.415923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['text_processed'] = text_processed","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:42.418380Z","iopub.execute_input":"2022-07-06T16:29:42.418684Z","iopub.status.idle":"2022-07-06T16:29:42.427230Z","shell.execute_reply.started":"2022-07-06T16:29:42.418648Z","shell.execute_reply":"2022-07-06T16:29:42.426456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-06T16:29:42.428625Z","iopub.execute_input":"2022-07-06T16:29:42.428889Z","iopub.status.idle":"2022-07-06T16:29:42.445833Z","shell.execute_reply.started":"2022-07-06T16:29:42.428854Z","shell.execute_reply":"2022-07-06T16:29:42.445005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now we need to change the categorical value of target(class) into numerical\n#train_data['discourse_type'] = train_data['discourse_type'].map({'Lead':0,'Position':1, 'Evidence':2, \n#                                                 'Claim':3, 'Concluding Statement':4,\n#                                    'Counterclaim':5,'Rebuttal':6 })\n#\n#train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:42.447349Z","iopub.execute_input":"2022-07-06T16:29:42.448033Z","iopub.status.idle":"2022-07-06T16:29:42.453785Z","shell.execute_reply.started":"2022-07-06T16:29:42.447996Z","shell.execute_reply":"2022-07-06T16:29:42.453065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split dataset into features and target\n\nX_features = train_data['text_processed'] \nY_target = train_data['discourse_type']\nprint('feature and target length: ', len(X_features), len(Y_target))\n\n# Now split the dataset into train and validation for training purpose\n\nfrom sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(X_features, Y_target,test_size=0.2, random_state=123,shuffle=True)\nlen(X_train), len(X_val), len(Y_train), len(Y_val)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:42.455224Z","iopub.execute_input":"2022-07-06T16:29:42.455615Z","iopub.status.idle":"2022-07-06T16:29:42.496890Z","shell.execute_reply.started":"2022-07-06T16:29:42.455507Z","shell.execute_reply":"2022-07-06T16:29:42.496092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_target","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:42.498241Z","iopub.execute_input":"2022-07-06T16:29:42.498683Z","iopub.status.idle":"2022-07-06T16:29:42.505853Z","shell.execute_reply.started":"2022-07-06T16:29:42.498649Z","shell.execute_reply":"2022-07-06T16:29:42.505110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets check if train and validation dataset follows similar disctirbutions\n\nplt.figure(figsize=(15,5))\nplt.subplot(1,3,1)\nsns.histplot(train_data['discourse_type'],bins=15)\nplt.subplot(1,3,2)\nplt.title(\"value counts of train dataset \")\nsns.histplot(Y_train,bins=15)\nplt.subplot(1,3,3)\nsns.histplot(Y_val,bins=15)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:42.507440Z","iopub.execute_input":"2022-07-06T16:29:42.507730Z","iopub.status.idle":"2022-07-06T16:29:43.657841Z","shell.execute_reply.started":"2022-07-06T16:29:42.507696Z","shell.execute_reply":"2022-07-06T16:29:43.657137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All three, the original dataset, train and validation have similar distribution, so we are good to go ahead for training","metadata":{}},{"cell_type":"code","source":"# Now we need to vecorize the string dataset to use it for training and validation \n# we use tfidf method of vectorization to find the bag of words\nfrom sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\ntfidf = TfidfVectorizer(max_features=None)\n\nbagofwords = tfidf.fit(X_train)\n\nbagofwords.vocabulary_","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:43.660616Z","iopub.execute_input":"2022-07-06T16:29:43.661092Z","iopub.status.idle":"2022-07-06T16:29:48.894969Z","shell.execute_reply.started":"2022-07-06T16:29:43.661035Z","shell.execute_reply":"2022-07-06T16:29:48.894098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(bagofwords.vocabulary_))   # length of bag of words\n# bag of words transformed\nbagofwords_tfm = bagofwords.transform(X_train)\nprint(bagofwords_tfm.shape)  # shape of final matrix after including all the documents of train dataset\nbagofwords_tfm_val = tfidf.transform(X_val)\nprint(bagofwords_tfm_val.shape) # shape of final matrix after including all the documents of validation dataset","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:48.896656Z","iopub.execute_input":"2022-07-06T16:29:48.896934Z","iopub.status.idle":"2022-07-06T16:29:56.173439Z","shell.execute_reply.started":"2022-07-06T16:29:48.896897Z","shell.execute_reply":"2022-07-06T16:29:56.172535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(bagofwords_tfm)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:56.174829Z","iopub.execute_input":"2022-07-06T16:29:56.175285Z","iopub.status.idle":"2022-07-06T16:29:56.211661Z","shell.execute_reply.started":"2022-07-06T16:29:56.175246Z","shell.execute_reply":"2022-07-06T16:29:56.210764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Classification model","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.naive_bayes import BernoulliNB, MultinomialNB\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score, f1_score, precision_score, recall_score","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:56.213185Z","iopub.execute_input":"2022-07-06T16:29:56.213505Z","iopub.status.idle":"2022-07-06T16:29:56.218724Z","shell.execute_reply.started":"2022-07-06T16:29:56.213455Z","shell.execute_reply":"2022-07-06T16:29:56.217659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets generalized the model by defining a helper function 'many_model' for few classification algorithms\nimport time\nimport datetime\ndef many_model(X,Y):\n    models = {}\n    training_time = {}\n    \n    #classifier = LogisticRegression(multi_class='multinomial', solver='lbfgs', max_iter=1000)\n    #classifier = BernoulliNB()\n    #classifier = MultinomialNB()\n    classifier = RandomForestClassifier(n_estimators = 100)\n    start_time = time.time()\n    classifier.fit(X,Y)\n    elapsed_time = datetime.timedelta(time.time() - start_time)\n    models.update({'model': classifier})\n    training_time.update({classifier: elapsed_time.seconds})\n    \n    return models, training_time\n     ","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:56.220599Z","iopub.execute_input":"2022-07-06T16:29:56.220939Z","iopub.status.idle":"2022-07-06T16:29:56.229255Z","shell.execute_reply.started":"2022-07-06T16:29:56.220904Z","shell.execute_reply":"2022-07-06T16:29:56.228452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate model\ndef evaluate_models(models, X_val, Y_val):\n    #scores_list = []\n    scores = {}\n    #class_wise_acc = {}\n    for model in models:\n        print(model)\n        prediction = models[model].predict(X_val)\n        # prediction = models[model].predict_prob(X_val)  for logistic regression (mulinomial)\n        accu = accuracy_score(Y_val, prediction)\n        f_score = f1_score(Y_val, prediction,average='macro')\n        precision = precision_score(Y_val, prediction,average='macro')\n        recall = recall_score(Y_val, prediction,average='macro')\n        \n        scores.update({model:[accu, f_score, precision, recall]})\n        #scores_list.append(scores)\n    return scores\n        ","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:56.230454Z","iopub.execute_input":"2022-07-06T16:29:56.230993Z","iopub.status.idle":"2022-07-06T16:29:56.239320Z","shell.execute_reply.started":"2022-07-06T16:29:56.230957Z","shell.execute_reply":"2022-07-06T16:29:56.238405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models, training_time = many_model(bagofwords_tfm,Y_train)\n\nmodels, training_time","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:29:56.241476Z","iopub.execute_input":"2022-07-06T16:29:56.241795Z","iopub.status.idle":"2022-07-06T16:40:23.358934Z","shell.execute_reply.started":"2022-07-06T16:29:56.241759Z","shell.execute_reply":"2022-07-06T16:40:23.358247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models['model']","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:05.52198Z","iopub.execute_input":"2022-03-15T18:50:05.527577Z","iopub.status.idle":"2022-03-15T18:50:05.548067Z","shell.execute_reply.started":"2022-03-15T18:50:05.527519Z","shell.execute_reply":"2022-03-15T18:50:05.547225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_models(models,bagofwords_tfm_val, Y_val )","metadata":{"execution":{"iopub.status.busy":"2022-07-06T16:40:23.360273Z","iopub.execute_input":"2022-07-06T16:40:23.360547Z","iopub.status.idle":"2022-07-06T16:40:26.625867Z","shell.execute_reply.started":"2022-07-06T16:40:23.360509Z","shell.execute_reply":"2022-07-06T16:40:26.625129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test data processing and making predition","metadata":{}},{"cell_type":"markdown","source":"1. Since we have to make prediction on each sentence.\n2. the important factor here are splitting the paragraph into sentences and make prediction on them.\n3. Another thing is finding the 'predictionstring' which typically is the number id of each word sequence","metadata":{}},{"cell_type":"code","source":"# read test files and store in dataframe\ntest_file_path = \"../input/feedback-prize-2021/test\"\n","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.824043Z","iopub.execute_input":"2022-03-15T18:50:06.824321Z","iopub.status.idle":"2022-03-15T18:50:06.828258Z","shell.execute_reply.started":"2022-03-15T18:50:06.824281Z","shell.execute_reply":"2022-03-15T18:50:06.827582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting the dataframe of test data from test files\n\ntest_ids = [ids[:-4] for ids in os.listdir(test_file_path)]  \ntest_ids\n\ntest_data = []\nfor test_id in test_ids:\n    #print(test_id)\n    path = test_file_path+'/' + test_id +'.txt'\n    #print(path)\n    sentence_id = []\n    initial_id = 0\n    with open(path, 'r') as txt: \n        text = txt.read()\n        sentences = nltk.sent_tokenize(text)\n\n        for sentence in sentences:\n            word_id = []\n            words = sentence.split()\n            for word in words:\n                word_id.append(initial_id)\n                initial_id+=1\n            sentence_id.append(word_id)\n            \n    test_data += list(zip([test_id]*len(sentences), sentences, sentence_id))\n    \ntest_df = pd.DataFrame(test_data, columns=['id', 'Discourse_text', 'prediction_str'])","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.82952Z","iopub.execute_input":"2022-03-15T18:50:06.82997Z","iopub.status.idle":"2022-03-15T18:50:06.858137Z","shell.execute_reply.started":"2022-03-15T18:50:06.829929Z","shell.execute_reply":"2022-03-15T18:50:06.85751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.859401Z","iopub.execute_input":"2022-03-15T18:50:06.859665Z","iopub.status.idle":"2022-03-15T18:50:06.871988Z","shell.execute_reply.started":"2022-03-15T18:50:06.859627Z","shell.execute_reply":"2022-03-15T18:50:06.871054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make predictionstring in the same format of train dataset\ntest_df['predictionstring'] = test_df['prediction_str'].apply(lambda x: \" \".join([str(i) for i in x]))\ntest_df.drop('prediction_str',axis = 1, inplace=True)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.87375Z","iopub.execute_input":"2022-03-15T18:50:06.874061Z","iopub.status.idle":"2022-03-15T18:50:06.891696Z","shell.execute_reply.started":"2022-03-15T18:50:06.874023Z","shell.execute_reply":"2022-03-15T18:50:06.890851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We preprocess test datset\n# we can use helper funtion cleanup_text(text) to clean the test datset and clened text is added as a column of test dataset\ntest_processed = test_df['Discourse_text'].apply(cleanup_text)\ntest_processed\ntest_df['text_processed'] =  test_processed\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.8932Z","iopub.execute_input":"2022-03-15T18:50:06.893471Z","iopub.status.idle":"2022-03-15T18:50:06.906544Z","shell.execute_reply.started":"2022-03-15T18:50:06.893434Z","shell.execute_reply":"2022-03-15T18:50:06.905765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# choose the test feature for transforming into vector and making prediction\ntest_feature = test_df['text_processed']\nbagofwords_tfm_test = tfidf.transform(test_feature)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.907755Z","iopub.execute_input":"2022-03-15T18:50:06.9083Z","iopub.status.idle":"2022-03-15T18:50:06.920879Z","shell.execute_reply.started":"2022-03-15T18:50:06.908261Z","shell.execute_reply":"2022-03-15T18:50:06.920188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for model in models:\ntest_predictions = models['model'].predict(bagofwords_tfm_test)\n\n# after prediction is made, it is converted into dataframe and the numerical class is changed into categorical as it was earlier\n#test_predictions= pd.DataFrame(test_predictions,columns=['prediction'])\n#test_predictions['prediction'] = test_predictions['prediction'].map({0:'Lead',1:'Position', 2:'Evidence', \n#                                             3:'Claim', 4:'Concluding Statement',\n#                                    5:'Counterclaim',6:'Rebuttal' }) \ntest_predictions","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-03-15T18:50:06.921915Z","iopub.execute_input":"2022-03-15T18:50:06.922174Z","iopub.status.idle":"2022-03-15T18:50:06.933305Z","shell.execute_reply.started":"2022-03-15T18:50:06.922138Z","shell.execute_reply":"2022-03-15T18:50:06.932338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred= pd.DataFrame(test_predictions,columns=['prediction'])\ntest_pred[0:20]","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.934269Z","iopub.execute_input":"2022-03-15T18:50:06.934466Z","iopub.status.idle":"2022-03-15T18:50:06.947267Z","shell.execute_reply.started":"2022-03-15T18:50:06.934435Z","shell.execute_reply":"2022-03-15T18:50:06.946396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# form the dataframe including id, class and prediction string\n#test_df['prediction'] = pd.DataFrame([test_predictions]) # added as a column to test_df \n#test_df.drop(['text_processed','Discourse_text'],axis=1, inplace=True) # unrequired columns are removed\n#test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.94884Z","iopub.execute_input":"2022-03-15T18:50:06.949248Z","iopub.status.idle":"2022-03-15T18:50:06.955811Z","shell.execute_reply.started":"2022-03-15T18:50:06.949211Z","shell.execute_reply":"2022-03-15T18:50:06.955049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare the submisiion file in the same format as given\nsubmission = pd.DataFrame()\nsubmission['id'] = test_df['id']\nsubmission['class'] = test_pred['prediction']\nsubmission['predictionstring'] = test_df['predictionstring']\n\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.957378Z","iopub.execute_input":"2022-03-15T18:50:06.957675Z","iopub.status.idle":"2022-03-15T18:50:06.975756Z","shell.execute_reply.started":"2022-03-15T18:50:06.957637Z","shell.execute_reply":"2022-03-15T18:50:06.974977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save to submission.csv to be uploaded as the prediction result.\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T18:50:06.977333Z","iopub.execute_input":"2022-03-15T18:50:06.97762Z","iopub.status.idle":"2022-03-15T18:50:06.986022Z","shell.execute_reply.started":"2022-03-15T18:50:06.97758Z","shell.execute_reply":"2022-03-15T18:50:06.985397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}