{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-08-12T08:59:20.127185Z","iopub.execute_input":"2022-08-12T08:59:20.127622Z","iopub.status.idle":"2022-08-12T08:59:20.140162Z","shell.execute_reply.started":"2022-08-12T08:59:20.127576Z","shell.execute_reply":"2022-08-12T08:59:20.138842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install langdetect","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:20.142784Z","iopub.execute_input":"2022-08-12T08:59:20.143285Z","iopub.status.idle":"2022-08-12T08:59:31.202658Z","shell.execute_reply.started":"2022-08-12T08:59:20.143224Z","shell.execute_reply":"2022-08-12T08:59:31.201491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.naive_bayes import MultinomialNB ##binary ise bernaulli ve gauss\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn import metrics\nfrom textblob import TextBlob, Word\nfrom nltk.stem.snowball import SnowballStemmer\n\nfrom langdetect import detect\n\n#Bulut resmi için\nimport os\nfrom PIL import Image\n\nfrom nltk.corpus import stopwords\nfrom stop_words import get_stop_words\nimport re\n#WordCloud\nfrom wordcloud import WordCloud,STOPWORDS\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:31.204926Z","iopub.execute_input":"2022-08-12T08:59:31.205291Z","iopub.status.idle":"2022-08-12T08:59:31.226278Z","shell.execute_reply.started":"2022-08-12T08:59:31.205256Z","shell.execute_reply":"2022-08-12T08:59:31.225153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/nlp-getting-started/sample_submission.csv') ","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:31.229286Z","iopub.execute_input":"2022-08-12T08:59:31.230040Z","iopub.status.idle":"2022-08-12T08:59:31.261898Z","shell.execute_reply.started":"2022-08-12T08:59:31.229992Z","shell.execute_reply":"2022-08-12T08:59:31.260634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:31.263665Z","iopub.execute_input":"2022-08-12T08:59:31.264413Z","iopub.status.idle":"2022-08-12T08:59:31.278557Z","shell.execute_reply.started":"2022-08-12T08:59:31.264366Z","shell.execute_reply":"2022-08-12T08:59:31.277146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/nlp-getting-started/test.csv')\ntrain = pd.read_csv('../input/nlp-getting-started/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:31.280346Z","iopub.execute_input":"2022-08-12T08:59:31.281363Z","iopub.status.idle":"2022-08-12T08:59:31.334839Z","shell.execute_reply.started":"2022-08-12T08:59:31.281320Z","shell.execute_reply":"2022-08-12T08:59:31.333837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:31.336506Z","iopub.execute_input":"2022-08-12T08:59:31.337306Z","iopub.status.idle":"2022-08-12T08:59:31.352806Z","shell.execute_reply.started":"2022-08-12T08:59:31.337259Z","shell.execute_reply":"2022-08-12T08:59:31.351755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:31.354373Z","iopub.execute_input":"2022-08-12T08:59:31.355538Z","iopub.status.idle":"2022-08-12T08:59:31.368932Z","shell.execute_reply.started":"2022-08-12T08:59:31.355489Z","shell.execute_reply":"2022-08-12T08:59:31.367603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['text'].replace({'r\"\\&\\#[0-9]+\\;\"':''},regex=True,inplace=True)#html karakter\ntrain['text'] = train['text'].str.lower()\ntrain['text'].str.replace('[^\\w\\s]','')#noktalama\ntrain['text'].str.replace('#','')\ntrain['text'].str.replace('\\d+','') #rakamları kaldırır\ntrain['text'].str.replace('\\n',' ').replace('\\r','')\ntrain['text'].replace({'r\"[\\s]+\"':''},regex=True,inplace=True)#gereksiz bosluk","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:31.370412Z","iopub.execute_input":"2022-08-12T08:59:31.371758Z","iopub.status.idle":"2022-08-12T08:59:31.478602Z","shell.execute_reply.started":"2022-08-12T08:59:31.371720Z","shell.execute_reply":"2022-08-12T08:59:31.477422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **LangDetect**","metadata":{}},{"cell_type":"code","source":"train['language'] = train['text'].apply(detect)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:59:31.484128Z","iopub.execute_input":"2022-08-12T08:59:31.484501Z","iopub.status.idle":"2022-08-12T09:00:30.651383Z","shell.execute_reply.started":"2022-08-12T08:59:31.484466Z","shell.execute_reply":"2022-08-12T09:00:30.650148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['language']=='ca'].head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:30.652726Z","iopub.execute_input":"2022-08-12T09:00:30.653082Z","iopub.status.idle":"2022-08-12T09:00:30.670923Z","shell.execute_reply.started":"2022-08-12T09:00:30.653049Z","shell.execute_reply":"2022-08-12T09:00:30.669927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['language'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:30.673811Z","iopub.execute_input":"2022-08-12T09:00:30.674495Z","iopub.status.idle":"2022-08-12T09:00:30.684006Z","shell.execute_reply.started":"2022-08-12T09:00:30.674461Z","shell.execute_reply":"2022-08-12T09:00:30.682961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[train['language']=='en']","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:30.685258Z","iopub.execute_input":"2022-08-12T09:00:30.685793Z","iopub.status.idle":"2022-08-12T09:00:30.698945Z","shell.execute_reply.started":"2022-08-12T09:00:30.685755Z","shell.execute_reply":"2022-08-12T09:00:30.697957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Tokenization**","metadata":{}},{"cell_type":"code","source":"from textblob import TextBlob\ndef detect_sentiment(text):\n      return TextBlob(text).sentiment.polarity","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:30.700288Z","iopub.execute_input":"2022-08-12T09:00:30.700843Z","iopub.status.idle":"2022-08-12T09:00:30.710404Z","shell.execute_reply.started":"2022-08-12T09:00:30.700809Z","shell.execute_reply":"2022-08-12T09:00:30.709254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['sentiment'] = train['text'].apply(detect_sentiment)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:30.711983Z","iopub.execute_input":"2022-08-12T09:00:30.712786Z","iopub.status.idle":"2022-08-12T09:00:32.643471Z","shell.execute_reply.started":"2022-08-12T09:00:30.712751Z","shell.execute_reply":"2022-08-12T09:00:32.642227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:32.645000Z","iopub.execute_input":"2022-08-12T09:00:32.645430Z","iopub.status.idle":"2022-08-12T09:00:32.660069Z","shell.execute_reply.started":"2022-08-12T09:00:32.645399Z","shell.execute_reply":"2022-08-12T09:00:32.658846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nnltk.download('punkt')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:32.661754Z","iopub.execute_input":"2022-08-12T09:00:32.662271Z","iopub.status.idle":"2022-08-12T09:00:32.945149Z","shell.execute_reply.started":"2022-08-12T09:00:32.662228Z","shell.execute_reply":"2022-08-12T09:00:32.944008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk import word_tokenize\ntrain['text'] = train['text'].apply(lambda x: word_tokenize(x.lower()))","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:32.946844Z","iopub.execute_input":"2022-08-12T09:00:32.947484Z","iopub.status.idle":"2022-08-12T09:00:34.532366Z","shell.execute_reply.started":"2022-08-12T09:00:32.947441Z","shell.execute_reply":"2022-08-12T09:00:34.531198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **StopWords Removal**","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords\nstop_words = list(stopwords.words('english'))\ntrain['text'] = train['text'].apply(lambda x: [word for word in x if word not in stop_words])","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:34.533620Z","iopub.execute_input":"2022-08-12T09:00:34.533977Z","iopub.status.idle":"2022-08-12T09:00:35.231627Z","shell.execute_reply.started":"2022-08-12T09:00:34.533945Z","shell.execute_reply":"2022-08-12T09:00:35.230626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:35.232881Z","iopub.execute_input":"2022-08-12T09:00:35.233778Z","iopub.status.idle":"2022-08-12T09:00:35.249336Z","shell.execute_reply.started":"2022-08-12T09:00:35.233741Z","shell.execute_reply":"2022-08-12T09:00:35.248220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['text'] = train['text'].apply(lambda x: ' '.join(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:35.250539Z","iopub.execute_input":"2022-08-12T09:00:35.251731Z","iopub.status.idle":"2022-08-12T09:00:35.271227Z","shell.execute_reply.started":"2022-08-12T09:00:35.251684Z","shell.execute_reply":"2022-08-12T09:00:35.270259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:35.272781Z","iopub.execute_input":"2022-08-12T09:00:35.273920Z","iopub.status.idle":"2022-08-12T09:00:35.287619Z","shell.execute_reply.started":"2022-08-12T09:00:35.273860Z","shell.execute_reply":"2022-08-12T09:00:35.286482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q wordcloud\nimport wordcloud\n\nimport nltk\nnltk.download('stopwords')\nnltk.download('wordnet')\nnltk.download('punkt')\nnltk.download('averaged_perceptron_tagger') \n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport io\nimport unicodedata\nimport numpy as np\nimport re\nimport string","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:35.289285Z","iopub.execute_input":"2022-08-12T09:00:35.290210Z","iopub.status.idle":"2022-08-12T09:00:46.421759Z","shell.execute_reply.started":"2022-08-12T09:00:35.290174Z","shell.execute_reply":"2022-08-12T09:00:46.420453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Lemmatization**","metadata":{}},{"cell_type":"code","source":"import nltk\nnltk.download('omw-1.4')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:46.423452Z","iopub.execute_input":"2022-08-12T09:00:46.423813Z","iopub.status.idle":"2022-08-12T09:00:46.671489Z","shell.execute_reply.started":"2022-08-12T09:00:46.423776Z","shell.execute_reply":"2022-08-12T09:00:46.670200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.stem import WordNetLemmatizer\ntrain['text'] = train['text'].apply(lambda x: WordNetLemmatizer().lemmatize(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:46.672696Z","iopub.execute_input":"2022-08-12T09:00:46.673499Z","iopub.status.idle":"2022-08-12T09:00:48.640152Z","shell.execute_reply.started":"2022-08-12T09:00:46.673457Z","shell.execute_reply":"2022-08-12T09:00:48.638894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['subjectivity'] = train['text'].apply(lambda x: TextBlob(x).sentiment.subjectivity)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:48.641656Z","iopub.execute_input":"2022-08-12T09:00:48.642817Z","iopub.status.idle":"2022-08-12T09:00:50.421082Z","shell.execute_reply.started":"2022-08-12T09:00:48.642773Z","shell.execute_reply":"2022-08-12T09:00:50.419966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.422416Z","iopub.execute_input":"2022-08-12T09:00:50.423094Z","iopub.status.idle":"2022-08-12T09:00:50.446421Z","shell.execute_reply.started":"2022-08-12T09:00:50.423056Z","shell.execute_reply":"2022-08-12T09:00:50.445525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[train['sentiment']>0.2,'sentiment']=2 #pozitive\ntrain.loc[train['sentiment']==0.2,'sentiment']=1 #neutral\ntrain.loc[train['sentiment']<0.2,'sentiment']=0 #negative  ","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.454105Z","iopub.execute_input":"2022-08-12T09:00:50.454672Z","iopub.status.idle":"2022-08-12T09:00:50.462629Z","shell.execute_reply.started":"2022-08-12T09:00:50.454638Z","shell.execute_reply":"2022-08-12T09:00:50.461497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[train['subjectivity']>0.2,'subjectivity']=2 #pozitive\ntrain.loc[train['subjectivity']==0.2,'subjectivity']=1 #neutral\ntrain.loc[train['subjectivity']<0.2,'subjectivity']=0 #negative","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.464247Z","iopub.execute_input":"2022-08-12T09:00:50.465382Z","iopub.status.idle":"2022-08-12T09:00:50.475250Z","shell.execute_reply.started":"2022-08-12T09:00:50.465337Z","shell.execute_reply":"2022-08-12T09:00:50.474098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sentiment.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.476640Z","iopub.execute_input":"2022-08-12T09:00:50.477197Z","iopub.status.idle":"2022-08-12T09:00:50.490365Z","shell.execute_reply.started":"2022-08-12T09:00:50.477154Z","shell.execute_reply":"2022-08-12T09:00:50.489486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.subjectivity.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.493398Z","iopub.execute_input":"2022-08-12T09:00:50.494046Z","iopub.status.idle":"2022-08-12T09:00:50.503549Z","shell.execute_reply.started":"2022-08-12T09:00:50.494011Z","shell.execute_reply":"2022-08-12T09:00:50.502377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = train[['text','subjectivity','sentiment','target']]","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.505034Z","iopub.execute_input":"2022-08-12T09:00:50.505413Z","iopub.status.idle":"2022-08-12T09:00:50.516309Z","shell.execute_reply.started":"2022-08-12T09:00:50.505376Z","shell.execute_reply":"2022-08-12T09:00:50.515048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = df1['text']\ny = df1['target']","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.518035Z","iopub.execute_input":"2022-08-12T09:00:50.519179Z","iopub.status.idle":"2022-08-12T09:00:50.526358Z","shell.execute_reply.started":"2022-08-12T09:00:50.519140Z","shell.execute_reply":"2022-08-12T09:00:50.525313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train,x_test,y_train,y_test = train_test_split(x,y, test_size=.3, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.527762Z","iopub.execute_input":"2022-08-12T09:00:50.528783Z","iopub.status.idle":"2022-08-12T09:00:50.541607Z","shell.execute_reply.started":"2022-08-12T09:00:50.528737Z","shell.execute_reply":"2022-08-12T09:00:50.540612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **TF-IDF**","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nvec=CountVectorizer(stop_words='english')\nx_train_vec=vec.fit_transform(x_train)\nx_test_vec=vec.transform(x_test)\nx_train_vec.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.543073Z","iopub.execute_input":"2022-08-12T09:00:50.544345Z","iopub.status.idle":"2022-08-12T09:00:50.704445Z","shell.execute_reply.started":"2022-08-12T09:00:50.544305Z","shell.execute_reply":"2022-08-12T09:00:50.703172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB,GaussianNB,BernoulliNB\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import classification_report,accuracy_score,confusion_matrix,precision_score,recall_score,f1_score","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:50.705673Z","iopub.execute_input":"2022-08-12T09:00:50.706015Z","iopub.status.idle":"2022-08-12T09:00:51.046710Z","shell.execute_reply.started":"2022-08-12T09:00:50.705985Z","shell.execute_reply":"2022-08-12T09:00:51.045480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr=LogisticRegression()\npred_vec=lr.fit(x_train_vec,y_train).predict(x_test_vec)\naccuracy=accuracy_score(y_test,pred_vec)\naccuracy","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:51.048177Z","iopub.execute_input":"2022-08-12T09:00:51.048594Z","iopub.status.idle":"2022-08-12T09:00:51.621204Z","shell.execute_reply.started":"2022-08-12T09:00:51.048561Z","shell.execute_reply":"2022-08-12T09:00:51.619917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"precision=precision_score(y_test,pred_vec,average='weighted')\nprecision","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:51.627513Z","iopub.execute_input":"2022-08-12T09:00:51.628407Z","iopub.status.idle":"2022-08-12T09:00:51.648853Z","shell.execute_reply.started":"2022-08-12T09:00:51.628355Z","shell.execute_reply":"2022-08-12T09:00:51.647677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1=f1_score(y_test,pred_vec,average='weighted')\nf1","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:51.650587Z","iopub.execute_input":"2022-08-12T09:00:51.651343Z","iopub.status.idle":"2022-08-12T09:00:51.670511Z","shell.execute_reply.started":"2022-08-12T09:00:51.651299Z","shell.execute_reply":"2022-08-12T09:00:51.669318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recall=recall_score(y_test,pred_vec,average='weighted')\nrecall","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:51.672444Z","iopub.execute_input":"2022-08-12T09:00:51.673788Z","iopub.status.idle":"2022-08-12T09:00:51.693482Z","shell.execute_reply.started":"2022-08-12T09:00:51.673728Z","shell.execute_reply":"2022-08-12T09:00:51.692273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix(y_test,pred_vec)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:51.695231Z","iopub.execute_input":"2022-08-12T09:00:51.696545Z","iopub.status.idle":"2022-08-12T09:00:51.720596Z","shell.execute_reply.started":"2022-08-12T09:00:51.696496Z","shell.execute_reply":"2022-08-12T09:00:51.718710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(confusion_matrix(y_test,pred_vec),annot=True);","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:51.728389Z","iopub.execute_input":"2022-08-12T09:00:51.729339Z","iopub.status.idle":"2022-08-12T09:00:51.981024Z","shell.execute_reply.started":"2022-08-12T09:00:51.729289Z","shell.execute_reply":"2022-08-12T09:00:51.979808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **N-GRAM**","metadata":{}},{"cell_type":"code","source":"vec2=CountVectorizer(ngram_range=(1,2),stop_words='english',binary=True)\nx_train_vec2=vec2.fit_transform(x_train)\nx_test_vec2=vec2.transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:51.983067Z","iopub.execute_input":"2022-08-12T09:00:51.983590Z","iopub.status.idle":"2022-08-12T09:00:52.351288Z","shell.execute_reply.started":"2022-08-12T09:00:51.983543Z","shell.execute_reply":"2022-08-12T09:00:52.350156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr=LogisticRegression()\npred_vec2=lr.fit(x_train_vec2,y_train).predict(x_test_vec2)\naccuracy=accuracy_score(y_test,pred_vec)\naccuracy","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:52.353248Z","iopub.execute_input":"2022-08-12T09:00:52.353696Z","iopub.status.idle":"2022-08-12T09:00:53.607133Z","shell.execute_reply.started":"2022-08-12T09:00:52.353652Z","shell.execute_reply":"2022-08-12T09:00:53.605830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"precision=precision_score(y_test,pred_vec2,average='micro')\nprecision","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:53.613138Z","iopub.execute_input":"2022-08-12T09:00:53.616313Z","iopub.status.idle":"2022-08-12T09:00:53.635367Z","shell.execute_reply.started":"2022-08-12T09:00:53.616252Z","shell.execute_reply":"2022-08-12T09:00:53.634112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1=f1_score(y_test,pred_vec2,average='weighted')\nf1","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:53.641408Z","iopub.execute_input":"2022-08-12T09:00:53.644794Z","iopub.status.idle":"2022-08-12T09:00:53.670760Z","shell.execute_reply.started":"2022-08-12T09:00:53.644731Z","shell.execute_reply":"2022-08-12T09:00:53.669109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recall=recall_score(y_test,pred_vec2,average='weighted')\nrecall","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:53.673954Z","iopub.execute_input":"2022-08-12T09:00:53.674839Z","iopub.status.idle":"2022-08-12T09:00:53.688056Z","shell.execute_reply.started":"2022-08-12T09:00:53.674790Z","shell.execute_reply":"2022-08-12T09:00:53.686551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix(y_test,pred_vec2)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:53.690454Z","iopub.execute_input":"2022-08-12T09:00:53.691297Z","iopub.status.idle":"2022-08-12T09:00:53.702554Z","shell.execute_reply.started":"2022-08-12T09:00:53.691253Z","shell.execute_reply":"2022-08-12T09:00:53.701233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(confusion_matrix(y_test,pred_vec2),annot=True);","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:53.706305Z","iopub.execute_input":"2022-08-12T09:00:53.708854Z","iopub.status.idle":"2022-08-12T09:00:53.919935Z","shell.execute_reply.started":"2022-08-12T09:00:53.708806Z","shell.execute_reply":"2022-08-12T09:00:53.918958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr=LogisticRegression()\npred_vec2=lr.fit(x_train_vec2,y_train).predict(x_test_vec2)\naccuracy=accuracy_score(y_test,pred_vec)\naccuracy","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:53.921538Z","iopub.execute_input":"2022-08-12T09:00:53.922649Z","iopub.status.idle":"2022-08-12T09:00:54.915463Z","shell.execute_reply.started":"2022-08-12T09:00:53.922609Z","shell.execute_reply":"2022-08-12T09:00:54.914218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Make a Submission**","metadata":{}},{"cell_type":"code","source":"test['text'].replace({'r\"\\&\\#[0-9]+\\;\"':''},regex=True,inplace=True)#html karakter\ntest['text'] = test['text'].str.lower()\ntest['text'].str.replace('[^\\w\\s]','')#noktalama\ntest['text'].str.replace('#','')\ntest['text'].str.replace('\\d+','') #rakamları kaldırır\ntest['text'].str.replace('\\n',' ').replace('\\r','')\ntest['text'].replace({'r\"[\\s]+\"':''},regex=True,inplace=True)#gereksiz bosluk","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:54.918233Z","iopub.execute_input":"2022-08-12T09:00:54.920272Z","iopub.status.idle":"2022-08-12T09:00:55.027460Z","shell.execute_reply.started":"2022-08-12T09:00:54.920218Z","shell.execute_reply":"2022-08-12T09:00:55.026207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['language'] = test['text'].apply(detect)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:00:55.029055Z","iopub.execute_input":"2022-08-12T09:00:55.029685Z","iopub.status.idle":"2022-08-12T09:01:21.159652Z","shell.execute_reply.started":"2022-08-12T09:00:55.029650Z","shell.execute_reply":"2022-08-12T09:01:21.158352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['language'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:21.161277Z","iopub.execute_input":"2022-08-12T09:01:21.161660Z","iopub.status.idle":"2022-08-12T09:01:21.171857Z","shell.execute_reply.started":"2022-08-12T09:01:21.161627Z","shell.execute_reply":"2022-08-12T09:01:21.170659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test[test['language']=='en']","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:21.173104Z","iopub.execute_input":"2022-08-12T09:01:21.173435Z","iopub.status.idle":"2022-08-12T09:01:21.184504Z","shell.execute_reply.started":"2022-08-12T09:01:21.173406Z","shell.execute_reply":"2022-08-12T09:01:21.183594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['text'] = test['text'].apply(lambda x: word_tokenize(x.lower()))","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:21.185992Z","iopub.execute_input":"2022-08-12T09:01:21.186776Z","iopub.status.idle":"2022-08-12T09:01:21.874635Z","shell.execute_reply.started":"2022-08-12T09:01:21.186740Z","shell.execute_reply":"2022-08-12T09:01:21.873646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['text'] = test['text'].apply(lambda x: [word for word in x if word not in stop_words])","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:21.876349Z","iopub.execute_input":"2022-08-12T09:01:21.877131Z","iopub.status.idle":"2022-08-12T09:01:22.047572Z","shell.execute_reply.started":"2022-08-12T09:01:21.877085Z","shell.execute_reply":"2022-08-12T09:01:22.046350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['text'] = test['text'].apply(lambda x: ' '.join(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.049388Z","iopub.execute_input":"2022-08-12T09:01:22.049980Z","iopub.status.idle":"2022-08-12T09:01:22.060492Z","shell.execute_reply.started":"2022-08-12T09:01:22.049930Z","shell.execute_reply":"2022-08-12T09:01:22.059359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['text'] = test['text'].apply(lambda x: WordNetLemmatizer().lemmatize(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.061898Z","iopub.execute_input":"2022-08-12T09:01:22.062491Z","iopub.status.idle":"2022-08-12T09:01:22.088807Z","shell.execute_reply.started":"2022-08-12T09:01:22.062455Z","shell.execute_reply":"2022-08-12T09:01:22.087932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.090193Z","iopub.execute_input":"2022-08-12T09:01:22.091333Z","iopub.status.idle":"2022-08-12T09:01:22.109250Z","shell.execute_reply.started":"2022-08-12T09:01:22.091289Z","shell.execute_reply":"2022-08-12T09:01:22.108020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test = test['text']","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.112588Z","iopub.execute_input":"2022-08-12T09:01:22.113574Z","iopub.status.idle":"2022-08-12T09:01:22.118648Z","shell.execute_reply.started":"2022-08-12T09:01:22.113526Z","shell.execute_reply":"2022-08-12T09:01:22.117711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = df1['text']\ny_train = df1['target']","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.119928Z","iopub.execute_input":"2022-08-12T09:01:22.120460Z","iopub.status.idle":"2022-08-12T09:01:22.130717Z","shell.execute_reply.started":"2022-08-12T09:01:22.120426Z","shell.execute_reply":"2022-08-12T09:01:22.129570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nvec=CountVectorizer(stop_words='english')\nx_train_vec=vec.fit_transform(x_train)\nx_test_vec=vec.transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.132670Z","iopub.execute_input":"2022-08-12T09:01:22.133126Z","iopub.status.idle":"2022-08-12T09:01:22.336112Z","shell.execute_reply.started":"2022-08-12T09:01:22.133081Z","shell.execute_reply":"2022-08-12T09:01:22.334888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr=LogisticRegression()\npred_vec=lr.fit(x_train_vec,y_train).predict(x_test_vec)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.337764Z","iopub.execute_input":"2022-08-12T09:01:22.338128Z","iopub.status.idle":"2022-08-12T09:01:22.825018Z","shell.execute_reply.started":"2022-08-12T09:01:22.338098Z","shell.execute_reply":"2022-08-12T09:01:22.823732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.826621Z","iopub.execute_input":"2022-08-12T09:01:22.827360Z","iopub.status.idle":"2022-08-12T09:01:22.834359Z","shell.execute_reply.started":"2022-08-12T09:01:22.827308Z","shell.execute_reply":"2022-08-12T09:01:22.832678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['id'] = test['id']","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.836264Z","iopub.execute_input":"2022-08-12T09:01:22.838072Z","iopub.status.idle":"2022-08-12T09:01:22.854377Z","shell.execute_reply.started":"2022-08-12T09:01:22.838021Z","shell.execute_reply":"2022-08-12T09:01:22.852789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['target'] = pred_vec","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.857556Z","iopub.execute_input":"2022-08-12T09:01:22.859426Z","iopub.status.idle":"2022-08-12T09:01:22.870370Z","shell.execute_reply.started":"2022-08-12T09:01:22.859370Z","shell.execute_reply":"2022-08-12T09:01:22.868912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.to_csv('result.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T09:01:22.873372Z","iopub.execute_input":"2022-08-12T09:01:22.875490Z","iopub.status.idle":"2022-08-12T09:01:22.903409Z","shell.execute_reply.started":"2022-08-12T09:01:22.875435Z","shell.execute_reply":"2022-08-12T09:01:22.901837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}