{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n# Text minig with Quora Insincere Questions Classification"},{"metadata":{},"cell_type":"markdown","source":"\n#### Sommaire\n\nI.\tANALYSE EXPLORATOIRE DES DONNEES\n\n\nII.\tPREPARATION DES DONNEES \n\n1.\tVectorization du texte\n\n    \n2.\tSelection des variables\n\nIII.\tMODELISATION \n\n\n    \n2.\tModèle selectionné\n\n3.\tHyperparameters tuning\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"%ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom zipfile import ZipFile\nfrom math import radians, cos, sin, asin, sqrt\nfrom datetime import datetime\nfrom nltk.stem.wordnet import WordNetLemmatizer\nfrom nltk.stem import PorterStemmer, SnowballStemmer\nfrom nltk import wordnet, pos_tag\nfrom nltk.tokenize import word_tokenize\nfrom nltk.util import ngrams\nfrom nltk.corpus import stopwords, wordnet as wn\nimport re\nimport string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\nsub_df_target = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = train.sample(frac=0.3,random_state=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test1 = test.sample(frac=0.3,random_state=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport numpy as np\nimport warnings                                  # `do not disturbe` mode\nwarnings.filterwarnings('ignore')\n\n#cora_data = pd.read_csv('all/train.csv')\n#cora_test_data = pd.read_csv('all/test.csv')\ncora_data = train1.copy()\ncora_test_data = test1.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# I. ANALYSE EXPLORATOIRE DES DONNEES"},{"metadata":{"trusted":true},"cell_type":"code","source":"cora_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Verification presence données manquantes\ncora_data[cora_data['question_text'].isnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Verification presence données manquantes\ncora_data[cora_data['target'].isnull()]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Nous avons zero données manquantes dans cette base de données."},{"metadata":{"trusted":true},"cell_type":"code","source":"percent_target = cora_data.groupby('target').count()\npercent_target['percent'] = 100*(percent_target['question_text']/cora_data['target'].count())\npercent_target.reset_index(level=0, inplace=True)\npercent_target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Pie chart, where the slices will be ordered and plotted counter-clockwise:\nlabels = 'Toxic contents','Positive Questions'\nsizes = [6, 94]\nexplode = (0.1, 0)  # only \"explode\" the 1st slice (i.e. 'Toxic contents')\n\nfig1, ax1 = plt.subplots()\nax1.pie(sizes, explode=explode, labels=labels, autopct='%1.0f%%',\n        shadow=True, startangle=90)\nax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **1. Echantillonnage**\n\nWe are in presence of a unbalenced data base. So we will try to increase the rate of the negative comments. (This way i also decrease the size of the database because my personnal computer don't have enought cpu to run the jobs on all the database.)\n\nThe final goal is to have 40% of negatives comments."},{"metadata":{"trusted":true},"cell_type":"code","source":"cora_data_neg_sample = cora_data[cora_data['target'] == 1] #Negatives comments\ncora_data_positive_sample = cora_data[cora_data['target'] == 0].reindex()  #Positive Comments\n\ncora_resampling = pd.concat([pd.DataFrame(cora_data_positive_sample.sample(6000)), #130000\n                               pd.DataFrame(cora_data_neg_sample.sample(3650))])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"100*(cora_resampling.groupby('target')['question_text'].count())/cora_resampling['target'].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Pie chart, where the slices will be ordered and plotted counter-clockwise:\nlabels = 'Toxic contents','Positive Questions'\nsizes = [38, 62]\nexplode = (0.1, 0)  # only \"explode\" the 1st slice (i.e. 'Toxic contents')\n\nfig1, ax1 = plt.subplots()\nax1.pie(sizes, explode=explode, labels=labels, autopct='%1.0f%%',\n        shadow=True, startangle=90)\nax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **2. Features engineering**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.stem.wordnet import WordNetLemmatizer\nfrom nltk.stem import PorterStemmer, SnowballStemmer\nfrom nltk import wordnet, pos_tag\nfrom nltk.tokenize import word_tokenize\nfrom nltk.util import ngrams\nfrom nltk.corpus import stopwords, wordnet as wn\nimport re\nimport string\n\n#Cleaning data\n\ndef clean_str(chaine):\n    chaine = re.sub(r\"[^A-Za-z0-9(),!?\\'\\`]\", \" \", chaine)     \n    chaine = re.sub(r\"\\'s\", \" \\'s\", chaine) \n    chaine = re.sub(r\"\\'ve\", \" \\'ve\", chaine) \n    chaine = re.sub(r\"n\\'t\", \" n\\'t\", chaine) \n    chaine = re.sub(r\"\\'re\", \" \\'re\", chaine) \n    chaine = re.sub(r\"\\'d\", \" \\'d\", chaine) \n    chaine = re.sub(r\"\\'ll\", \" \\'ll\", chaine) \n    chaine = re.sub(r\",\", \" , \", chaine) \n    chaine = re.sub(r\"!\", \" ! \", chaine) \n    chaine = re.sub(r\"\\(\", \" \\( \", chaine) \n    chaine = re.sub(r\"\\)\", \" \\) \", chaine) \n    chaine = re.sub(r\"\\?\", \" \\? \", chaine) \n    chaine = re.sub(r\"\\s{2,}\", \" \", chaine)\n    chaine = chaine.lower() #convert all text in lower case\n    chaine = chaine.replace(' +', ' ') # Remove double space\n    chaine = chaine.strip() # Remove trailing space at the beginning or end\n    chaine = chaine.replace('[^a-zA-Z]', ' ' )# Everything not a alphabet character replaced with a space\n    #words =  [word for word in chaine.split() if word not in [i for i in string.punctuation]] #Remove punctuations\n    words =  [word for word in chaine.split() if word.isalpha()] #droping numbers and punctuations\n    return ' '.join(words)\n\n#Tokenization and punctuation removing and stopwords\ndef tokeniZ_stopWords(chaine):\n    chaine = word_tokenize(chaine)\n    list_stopWords = set(stopwords.words('english'))\n    words = [word for word in chaine if word not in list_stopWords]\n    return words\n\n#Stemming \nps = PorterStemmer()\nsb = SnowballStemmer('english')\n\n#Lemmatization\ndef lemat_words(tokens_list):\n    from collections import defaultdict\n    tag_map = defaultdict(lambda : wn.NOUN)\n    tag_map['J'] = wn.ADJ\n    tag_map['V'] = wn.VERB\n    tag_map['R'] = wn.ADV\n    lemma_function = WordNetLemmatizer()\n    return [lemma_function.lemmatize(token, tag_map[tag[0]]) for token, tag in pos_tag(tokens_list)]\n    #for token, tag in pos_tag(tokens_list):\n     #   lemma = lemma_function.lemmatize(token, tag_map[tag[0]])\n\n# Define Ngrams function\ndef get_ngrams(text, n ):\n    n_grams = ngrams(word_tokenize(text), n)\n    return [ ' '.join(grams) for grams in n_grams]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Cleaning the data \ncora_resampling['clean_question'] = cora_resampling['question_text'].apply(clean_str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Tokenizing and stopwords removing\ncora_resampling['tokeniZ_stopWords_question'] = cora_resampling['clean_question'].apply(tokeniZ_stopWords)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Words Stemming\ncora_resampling['stemming_question'] = [[ps.stem(word) for word in words] for words in cora_resampling['tokeniZ_stopWords_question'] ]\ncora_resampling['stemming_question_for_tfidf'] = [' '.join(words) for words in cora_resampling['stemming_question']] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Words lemmatization\ncora_resampling['lemmatize_question'] = cora_resampling['tokeniZ_stopWords_question'].apply(lemat_words)\ncora_resampling['lemmatize_question_for_tfidf'] = [' '.join(x) for x in cora_resampling['lemmatize_question'] ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Calcul longueur des commentaires\ncora_resampling['question_lenght'] = cora_resampling['question_text'].apply(len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Calcul du nombre de ponctuation par question\nfrom string import punctuation\ncora_resampling['number_punctuation'] = cora_resampling['question_text'].apply(\n    lambda doc: len([word for word in str(doc) if word in punctuation])) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of unique words in the text\ncora_resampling['number_of_Unique_words'] = cora_resampling['clean_question'].apply([lambda x : len(set(str(x).split()))])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of stopwords in the text\nlist_stopWords = set(stopwords.words('english'))\ncora_resampling['number_of_StopWords'] = cora_resampling['clean_question'].apply(\n    lambda x : len([w for w in x.lower().split() if w in list_stopWords ]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of upper case words\ncora_resampling['number_of_uppercase'] = cora_resampling['question_text'].apply(\n    lambda x : len([w for w in x.split() if w.isupper()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Average length of words in the text (whithout stop words)\ncora_resampling['average_of_wordsLength'] = cora_resampling['clean_question'].apply(\n    lambda x : np.mean([len(w) for w in x.split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of words in the text\ncora_resampling['number_of_words'] = cora_resampling['clean_question'].apply([lambda x : len(str(x).split())])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cora_resampling.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cora_resampling[['question_lenght', 'number_punctuation', 'number_of_words',\n       'number_of_Unique_words', 'number_of_StopWords', 'number_of_uppercase',\n       'average_of_wordsLength']].sample(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **3. Data visualisation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_var=['question_lenght', 'number_punctuation', 'number_of_Unique_words', \n          'number_of_StopWords', 'number_of_uppercase', 'average_of_wordsLength']\ndef var_hist_global(df,X='target',Y=list_var, Title='Features Engineering - Histograms', KDE=False):\n    fig, ((ax1, ax2),(ax3,ax4),(ax5,ax6)) = plt.subplots(3, 2 ,figsize=(14,16))#, sharey=True )\n    aX = [ax1, ax2,ax3,ax4,ax5,ax6]\n    for i in range(len(list_var)):   \n        sns.distplot( df[list_var[i]][df[X]== 1 ].dropna(), label=\"unsinceres questions\" , ax= aX[i], kde= KDE , color = 'red')           \n        sns.distplot( df[list_var[i]][df[X]== 0 ].dropna(), label=\"Sinceres questions\"   , ax= aX[i], kde= KDE , color = \"olive\")\n    plt.legend()\n    plt.title(Title)\n    #plt.show()\n    plt.savefig(\"Features_Engineering_Histograms\")\n    \nvar_hist_global(df=cora_resampling,X='target',Y=list_var, Title='Histogramme Quora Questions', KDE=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate number of obs per group & median to position labels\nlist_var = ['question_lenght', 'number_of_Unique_words', 'number_of_StopWords']\ndef violin_boxplott(df,X='target',Y=list_var, Title='Features Engineering - Box plot'): \n    fig, (ax1, ax2 ,ax3) = plt.subplots(1,3 ,figsize=(14,8))#, sharey=True )\n    medians = cora_resampling.groupby(['target'])['question_lenght', 'number_of_Unique_words', 'number_of_StopWords'].median().values\n \n    sns.boxplot( y=list_var[0],  x=X , data = df, ax= ax1 , palette=['olive','red'])\n    sns.boxplot( y=list_var[1],  x=X , data = df, ax= ax2 , palette=['olive','red'])\n    sns.boxplot( y=list_var[2],  x=X , data = df, ax= ax3 , palette=['olive','red'])\n    #plt.title(Title)\n    plt.savefig(\"Features_Engineering_Boxplot\")\nviolin_boxplott(df=cora_resampling)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"On constate que les variables suivantes : \n    - number_punctuation, number_of_uppercase et average_of_wordsLength ne sont pas discriminante par rapport à la cible. \n    En effet, les distribution des questions sincères et pas sincières sont quasiment les memes. \n    Elles seront peu voir pas du tout significatives pour expliquer le modèle."},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\n# Code recuperer de : https://www.kaggle.com/aashita/word-clouds-of-various-shapes ##\ndef plot_wordcloud(text, mask=None, max_words=200, max_font_size=100, figure_size=(24.0,16.0), \n                   title = None, title_size=40, image_color=False):\n    stopwords = set(STOPWORDS)\n    more_stopwords = {'one', 'quora','br', 'Po', 'th', 'sayi', 'fo', 'Unknown','will','say','now','must','want','much','talks','buy','dont','use','etc','go','ago','lot','ki', 'ba'}\n    stopwords = stopwords.union(more_stopwords)\n\n    wordcloud = WordCloud(background_color='black',\n                    stopwords = stopwords,\n                    max_words = max_words,\n                    max_font_size = max_font_size, \n                    random_state = 42,\n                    width=800, \n                    height=400,\n                    mask = mask)\n    wordcloud.generate(str(text))\n    \n    plt.figure(figsize=figure_size)\n    if image_color:\n        image_colors = ImageColorGenerator(mask);\n        plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation=\"bilinear\");\n        plt.title(title, fontdict={'size': title_size,  \n                                  'verticalalignment': 'bottom'})\n    else:\n        plt.imshow(wordcloud);\n        plt.title(title, fontdict={'size': title_size, 'color': 'black', \n                                  'verticalalignment': 'bottom'})\n    plt.axis('off');\n    plt.savefig(title)\n    plt.tight_layout()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plot_wordcloud(cora_data_neg_sample[\"question_text\"], title=\"Word Cloud of insincere Questions\")\n#plot_wordcloud(cora_resampling['tokeniZ_stopWords_question'][cora_resampling['target']== 1]) \nplot_wordcloud(cora_resampling['lemmatize_question_for_tfidf'][cora_resampling['target']== 1], title=\"Word Cloud of insincere Questions\") ","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"#plot_wordcloud(cora_data_positive_sample[\"question_text\"], title=\"Word Cloud of sincere Questions\")\nplot_wordcloud(cora_resampling['lemmatize_question_for_tfidf'][cora_resampling['target']== 0], title=\"Word Cloud of sincere Questions\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"SI POSSIBLE Essayer de faire l'histogramme du T3gram et Bigram."},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# Code source : https://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-qiqc\nfrom collections import defaultdict\nfrom wordcloud import  STOPWORDS\nimport plotly.graph_objs as go\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\n\n#train1_df = train_df[train_df[\"target\"]==1]\n#train0_df = train_df[train_df[\"target\"]==0]\ntrain1_df = cora_resampling[cora_resampling['target']==1]\ntrain0_df = cora_resampling[cora_resampling['target']==0]\n## custom function for ngram generation ##\ndef generate_ngrams(text, n_gram=1):\n    token = [token for token in text.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n    ngrams = zip(*[token[i:] for i in range(n_gram)])\n    return [\" \".join(ngram) for ngram in ngrams]\n\n## custom function for horizontal bar chart ##\ndef horizontal_bar_chart(df, color):\n    trace = go.Bar(\n        y=df[\"word\"].values[::-1],\n        x=df[\"wordcount\"].values[::-1],\n        showlegend=False,\n        orientation = 'h',\n        marker=dict(\n            color=color,\n        ),\n    )\n    return trace\n\n## Get the bar chart from sincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train0_df[\"lemmatize_question_for_tfidf\"]:\n    for word in generate_ngrams(sent):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(20), 'blue')\n\n## Get the bar chart from insincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"lemmatize_question_for_tfidf\"]:\n    for word in generate_ngrams(sent):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,\n                          subplot_titles=[\"Frequent words of sincere questions\", \n                                          \"Frequent words of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title=\"Word Count Plots\")\npy.iplot(fig, filename='word-plots')\n\n#plt.figure(figsize=(10,16))\n#sns.barplot(x=\"ngram_count\", y=\"ngram\", data=fd_sorted.loc[:50,:], color=\"b\")\n#plt.title(\"Frequent words for Insincere Questions\", fontsize=16)\n#plt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"freq_dict = defaultdict(int)\nfor sent in train0_df[\"lemmatize_question_for_tfidf\"]:\n    for word in generate_ngrams(sent,2):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'orange')\n\n\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"lemmatize_question_for_tfidf\"]:\n    for word in generate_ngrams(sent,2):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(20), 'orange')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,horizontal_spacing=0.15,\n                          subplot_titles=[\"Frequent bigrams of sincere questions\", \n                                          \"Frequent bigrams of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title=\"Bigram Count Plots - N_gram(2,2)\")\npy.iplot(fig, filename='word-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_dict = defaultdict(int)\nfor sent in train0_df[\"lemmatize_question_for_tfidf\"]:\n    for word in generate_ngrams(sent,3):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'green')\n\n\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"lemmatize_question_for_tfidf\"]:\n    for word in generate_ngrams(sent,3):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(20), 'green')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04, horizontal_spacing=0.2,\n                          subplot_titles=[\"Frequent trigrams of sincere questions\", \n                                          \"Frequent trigrams of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=1200, paper_bgcolor='rgb(233,233,233)', title=\"Trigram Count Plots - N_gram(3,3)\")\npy.iplot(fig, filename='word-plots')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# II. PREPARATION DES DONNEES "},{"metadata":{"trusted":true},"cell_type":"code","source":"cora_resampling.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_cora_train, X_cora_test, y_cora_train, y_cora_test = train_test_split(\n            cora_resampling[['clean_question', 'stemming_question_for_tfidf', 'lemmatize_question_for_tfidf',\n                             'tokeniZ_stopWords_question', 'stemming_question', 'lemmatize_question',\n                             'question_lenght', 'number_punctuation', 'number_of_StopWords', 'number_of_Unique_words', 'number_of_uppercase','average_of_wordsLength']]\n            ,cora_resampling['target'], \n            test_size=0.3, random_state=42)\nX_cora_train.shape, X_cora_test.shape, y_cora_train.shape, y_cora_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **1. Vectorization des données**"},{"metadata":{},"cell_type":"markdown","source":"#### **1.1 Tf-Idf Vectorizer** "},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\ntfidf_vectorizer = TfidfVectorizer(  ngram_range=(1,1), \n                                     analyzer='word',\n                                     stop_words='english', \n                                     lowercase=True, \n                                     max_df=0.9, # remove too frequent words\n                                     min_df=10, # remove too rare words\n                                     max_features = None, # max words in vocabulary, will keep most frequent words\n                                     binary=False #If True, all non zero counts are set to 1. This is useful for discrete probabilistic models that model binary events rather than integer counts.\n                                  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Stemmed questions vectorzation\nX_tfidf_vectorizer_train = tfidf_vectorizer.fit_transform(X_cora_train['stemming_question_for_tfidf'])\nX_tfidf_vectorizer_test = tfidf_vectorizer.transform(X_cora_test['stemming_question_for_tfidf'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Lemmentized questions vectorization\nX_tfidf_Lem_vect_train = tfidf_vectorizer.fit_transform(X_cora_train['lemmatize_question_for_tfidf'])\nX_tfidf_Lem_vect_test = tfidf_vectorizer.transform(X_cora_test['lemmatize_question_for_tfidf'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#bigram questions vectorization\nbigram_vectorizer = TfidfVectorizer(  ngram_range=(1,2), \n                                     analyzer='word',\n                                     stop_words='english', \n                                     lowercase=True, \n                                     max_df=0.9, # remove too frequent words\n                                     min_df=10, # remove too rare words\n                                     max_features = None, # max words in vocabulary, will keep most frequent words\n                                     binary=False #If True, all non zero counts are set to 1. This is useful for discrete probabilistic models that model binary events rather than integer counts.\n                                  )\nX_bigram_vectorizer_train = bigram_vectorizer.fit_transform(X_cora_train['stemming_question_for_tfidf'])\nX_bigram_vectorizer_test = bigram_vectorizer.transform(X_cora_test['stemming_question_for_tfidf'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#T3gram questions vectorization\nt3gram_vectorizer = TfidfVectorizer(  ngram_range=(1,4), \n                                     analyzer='word',\n                                     stop_words='english', \n                                     lowercase=True, \n                                     max_df=0.9, # remove too frequent words\n                                     min_df=10, # remove too rare words\n                                     max_features = None, # max words in vocabulary, will keep most frequent words\n                                     binary=False #If True, all non zero counts are set to 1. This is useful for discrete probabilistic models that model binary events rather than integer counts.\n                                  )\nX_t3gram_vectorizer_train = t3gram_vectorizer.fit_transform(X_cora_train['stemming_question_for_tfidf'])\nX_t3gram_vectorizer_test = t3gram_vectorizer.transform(X_cora_test['stemming_question_for_tfidf'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Range single word to t3gram questions vectorization\nst3gram_vectorizer = TfidfVectorizer(  ngram_range=(1,3), \n                                     analyzer='word',\n                                     stop_words='english', \n                                     lowercase=True, \n                                     max_df=0.9, # remove too frequent words\n                                     min_df=10, # remove too rare words\n                                     max_features = None, # max words in vocabulary, will keep most frequent words\n                                     binary=False #If True, all non zero counts are set to 1. This is useful for discrete probabilistic models that model binary events rather than integer counts.\n                                  )\nX_Singt3gram_vectorizer_train = st3gram_vectorizer.fit_transform(X_cora_train['stemming_question_for_tfidf'])\nX_Singt3gram_vectorizer_test  = st3gram_vectorizer.transform(X_cora_test['stemming_question_for_tfidf'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_Singt3gram_vectorizer_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### **1.2 Word embedding - Doc2Vec**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Word2Vec with preprocessiong questions (without stopwords) \nfrom gensim.models.doc2vec import Doc2Vec, TaggedDocument\n\nd2v_training_data = []\nfor i, doc in enumerate(X_cora_train['stemming_question']):\n    d2v_training_data.append(TaggedDocument(words=doc,tags=[i]))\n\n# ========== learning doc embeddings with doc2vec ==========\n\n# PV stands for 'Paragraph Vector'\n# PV-DBOW (distributed bag-of-words) dm=0\n\nd2v = Doc2Vec(d2v_training_data, vector_size=300, window=10, alpha=0.1, min_alpha=1e-4, dm=0, negative=1, epochs=10, min_count=2, workers=4)\nd2v_vecs = np.zeros((len(X_cora_train['stemming_question']), 300))\nfor i in range(len(X_cora_train['stemming_question'])):\n    d2v_vecs[i,:] = d2v.docvecs[i]\n    \nd2v_test = np.zeros((len(X_cora_test['stemming_question']), 300))\nfor i in range(len(X_cora_test['stemming_question'])):\n    d2v_test[i,:] = d2v.infer_vector(X_cora_test['stemming_question'].iloc[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Word2Vec with lemmatize words\nd2v_training_data = []\nfor i, doc in enumerate(X_cora_train['lemmatize_question']):\n    d2v_training_data.append(TaggedDocument(words=doc,tags=[i]))\n\n# ========== learning doc embeddings with doc2vec ==========\n\n# PV stands for 'Paragraph Vector'\n# PV-DBOW (distributed bag-of-words) dm=0\n\nd2v = Doc2Vec(d2v_training_data, vector_size=200, window=5, alpha=0.1, min_alpha=1e-4, \n              dm=0, negative=1, epochs=10, min_count=2, workers=4)\nd2v_vecs_bigram = np.zeros((len(X_cora_train['lemmatize_question']), 200))\nfor i in range(len(X_cora_train['lemmatize_question'])):\n    d2v_vecs_bigram[i,:] = d2v.docvecs[i]\n    \nd2v_test_bigram = np.zeros((len(X_cora_test['lemmatize_question']), 200))\nfor i in range(len(X_cora_test['lemmatize_question'])):\n    d2v_test_bigram[i,:] = d2v.infer_vector(X_cora_test['lemmatize_question'].iloc[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **2. FEATURES SELECTION**\n\nFaisons un test non parametrique d'indépendance entre les variables quantitatives créées et la variable cible. \nH0 (indépendance entre une variable données et la varibale cible) est rejetée si la P_value <= 0.05 "},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest, chi2, f_classif, mutual_info_classif, SelectPercentile\nfrom sklearn.pipeline import Pipeline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = SelectKBest(mutual_info_classif,k=2).fit(X_cora_train[['question_lenght', 'number_punctuation', 'number_of_StopWords', 'number_of_Unique_words', \n                                        'number_of_uppercase','average_of_wordsLength']].fillna(0),y_cora_train)\nindependance_test = np.zeros((6,2))\nfor idx,i in enumerate(['question_lenght', 'number_punctuation', 'number_of_StopWords', 'number_of_Unique_words', 'number_of_uppercase','average_of_wordsLength']):\n    #independance_test[idx,0]= features.pvalues_[idx]\n    independance_test[idx,1]= features.scores_[idx]\n    #print (i,features.pvalues_[idx],features.scores_[idx])\n    #print('%s  %s'%(i,features.scores_[idx]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_var=['question_lenght', 'number_punctuation', 'number_of_StopWords', 'number_of_Unique_words', 'number_of_uppercase','average_of_wordsLength']\nindependance_df = pd.DataFrame({'Variables': list_var, 'p_values': independance_test[:,0], 'MI': independance_test[:,1]},index=None)\nindependance_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"On rejette H0 en faveur de l'hypothèse alternative (dépendance avec la variable cible) pour l'ensemble des variables textées."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 10))\n_ = sns.heatmap(cora_resampling[['question_lenght', 'number_punctuation', 'number_of_StopWords', \n                                 'number_of_Unique_words', 'number_of_uppercase','average_of_wordsLength']].corr()\n                ,cmap=\"YlGnBu\", annot=True, fmt=\".2f\")\nplt.savefig(\"Correlation Matrice\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Pour les matrices generées à partir des textes via tf_idf et Doc2Vec nous appliquerons un filtre sur les features si nécessaire dans la section qui suit (lors de la modélisation).**"},{"metadata":{},"cell_type":"markdown","source":"# III. MODELISATION\n\n\n### **1. Description du process de modélisation**\nNou testerons dans cette section plusieurs modèles de classification tels les SVM, les methodes ensembles (RF, adaboost,...), Reseaux de neurones etc... dans l'objectif de choisir le meilleur modèle. Puis nous optimiserons les hyperparametres des modèles qui nous semblent les plus performants. Enfin nous generons les courbes d'apprentissage afn d'évaluer le niveau d'apprentissage de ces modèles pour verifer l'overfitting où  l'underfitting. "},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score,roc_auc_score, f1_score, balanced_accuracy_score\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import learning_curve\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom matplotlib import pyplot as plt\nplt.style.use('seaborn-whitegrid')\n\ndef plot_learning_curve(estimator1, X, y, estimator2, ylim=(0, 1.1), cv=2, n_jobs=-1, \n                        train_sizes=np.linspace(.1, 1.0, 5), scoring=None):\n    \n    \n    plt.figure(figsize=(12,6))\n    #plt.title(\"Learning curves for %s\" % type(estimator1).__name__)\n    #plt.title(\"Learning curves for %s\" %(estimator1))\n    plt.grid()\n    plt.xlabel(\"Training examples\")\n    plt.ylabel(\"Score\")\n    plt.xscale('log')\n    train_sizes, train_scores, test_scores = learning_curve(\n        estimator1, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes,\n        scoring=scoring)\n    train_scores_mean = np.mean(train_scores, axis=1)\n    test_scores_mean = np.mean(test_scores, axis=1)\n    train_scores_std = np.std(train_scores, axis=1)\n    test_scores_std = np.std(test_scores, axis=1)\n    \n    plt.subplot(1, 2, 1)\n    plt.title(\"Learning curves for %s\" % type(estimator1).__name__)\n    plt.plot(\n        train_sizes, train_scores_mean, 'o-',\n        color=\"r\", #linewidth=3, \n        label=\"Training score\")\n    plt.plot(\n        train_sizes, test_scores_mean, 'o-',\n        color=\"olive\", \n        label=\"Cross-validation score\")\n    plt.fill_between(\n        train_sizes, train_scores_mean - train_scores_std,\n        train_scores_mean + train_scores_std, alpha=0.1,\n        color=\"firebrick\")\n    plt.fill_between(\n        train_sizes, test_scores_mean - test_scores_std,\n        test_scores_mean + test_scores_std, alpha=0.1, color=\"darkgoldenrod\")\n    plt.legend(loc=\"best\")\n    \n    train_sizes, train_scores, test_scores = learning_curve(\n        estimator2, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes,\n        scoring=scoring)\n    train_scores_mean = np.mean(train_scores, axis=1)\n    test_scores_mean = np.mean(test_scores, axis=1)\n    train_scores_std = np.std(train_scores, axis=1)\n    test_scores_std = np.std(test_scores, axis=1)\n    \n    plt.subplot(1, 2, 2)\n    plt.title(\"Learning curves for %s with 70 percent of best features\" % type(estimator1).__name__)\n    plt.plot(\n        train_sizes, train_scores_mean, 'o-',\n        color=\"r\", #linewidth=3, \n        label=\"Training score\")\n    plt.plot(\n        train_sizes, test_scores_mean, 'o-',\n        color=\"olive\", \n        label=\"Cross-validation score\")\n    plt.fill_between(\n        train_sizes, train_scores_mean - train_scores_std,\n        train_scores_mean + train_scores_std, alpha=0.1,\n        color=\"firebrick\")\n    plt.fill_between(\n        train_sizes, test_scores_mean - test_scores_std,\n        test_scores_mean + test_scores_std, alpha=0.1, color=\"darkgoldenrod\")\n    plt.legend(loc=\"best\")\n    plt.savefig(\"Learning curves for %s\" % type(estimator1).__name__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_models = pd.DataFrame({'Models':[], 'Sample':[], 'Accuracy':[],'Accuracy with 70% best features':[]})\ndata=[]\n\ndef modelize(list_clf,X,y,X_test,y_test):\n    selector = SelectPercentile(mutual_info_classif,percentile=70)\n    for clf in list_clf:\n        Clf1 = clf().fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf())]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=2 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted'),accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Training', 'Accuracy':accuracy_score(y,Clf1.predict(X)),'Accuracy with 70% best features':accuracy_score(y,Clf2.predict(X)) })\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Test', 'Accuracy':accuracy_score(y_test,Clf1.predict(X_test)),'Accuracy with 70% best features':accuracy_score(y_test,Clf2.predict(X_test)) })\n        #plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegressionCV , PassiveAggressiveClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier , ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier\nfrom sklearn.svm import NuSVC, LinearSVC, SVC, OneClassSVM\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.naive_bayes import GaussianNB, MultinomialNB, ComplementNB","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generalized_linear_model = [LogisticRegressionCV,PassiveAggressiveClassifier]\nsupport_vector_machines =  [NuSVC, LinearSVC]   \ndecisionTreeClassification=[DecisionTreeClassifier]\nensemble_methods = [RandomForestClassifier , ExtraTreesClassifier,AdaBoostClassifier, GradientBoostingClassifier]\nnaive_bayes_model = [GaussianNB, MultinomialNB, ComplementNB]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### **1.1 MODELISATION AVEC TF-IDF A PARTIR DES DONNEES SOUMISES A LA STEMMIZATION (1,1)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Generalized Linear Model\nmodelize(generalized_linear_model,X_tfidf_vectorizer_train,y_cora_train,X_tfidf_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**SVM For Unbalanced problems**\n\nIn problems where it is desired to give more importance to certain classes or certain individual samples keywords **class_weight** and **sample_weight** can be used.\n**SVC** (but not **NuSVC**) implement a keyword **class_weight** in the **fit** method. \nIt’s a dictionary of the form **{class_label : value}**, where value is a floating point number > 0 that sets the parameter **C** of class **class_label** to **C * value**."},{"metadata":{"trusted":true},"cell_type":"code","source":"def modelize_svc(list_clf,X,y,X_test,y_test):\n    selector = SelectPercentile(mutual_info_classif,percentile=70)\n    for clf in list_clf:\n        Clf1 = clf(kernel='poly').fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(kernel='poly'))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=2 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Training', 'Accuracy':accuracy_score(y_test,Clf1.predict(X_test))\n                       ,'Accuracy with 70% best features':accuracy_score(y_test,Clf2.predict(X_test)) })\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Test', 'Accuracy':accuracy_score(y_test,Clf1.predict(X_test))\n                       ,'Accuracy with 70% best features':accuracy_score(y_test,Clf2.predict(X_test)) })\n        #plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\n#Support Vector Machine\n#modelize_svc([SVC],X_tfidf_vectorizer_train,y_cora_train,X_tfidf_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Support Vector Machine\nmodelize(support_vector_machines,X_tfidf_vectorizer_train,y_cora_train,X_tfidf_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Decision Tree\nmodelize(decisionTreeClassification,X_tfidf_vectorizer_train,y_cora_train,X_tfidf_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Ensemble Methods\nmodelize(ensemble_methods,X_tfidf_vectorizer_train,y_cora_train,X_tfidf_vectorizer_test,y_cora_test)       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Naives bayes\nmodelize([ MultinomialNB, ComplementNB],X_tfidf_vectorizer_train,y_cora_train,X_tfidf_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Neural Network - Consomme beaucoup trop d'energie\nNeural_network= [MLPClassifier]\nmodelize(Neural_network,X_tfidf_vectorizer_train,y_cora_train,X_tfidf_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_df =pd.concat([pd.DataFrame(data),df_models],ignore_index=True)  \nmodels_df.sort_values(by=['Accuracy','Models','Sample'], ascending=False, axis=0, inplace=True)\ntest_df = pd.DataFrame(np.array(models_df[['Accuracy', 'Accuracy with 70% best features']]),\n                       index=[models_df['Models'],models_df['Sample']], columns=['Accuracy', 'Accuracy with 70% best features'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df1=pd.merge(models_df[models_df['Sample']=='Test'],models_df[models_df['Sample']=='Training'],how='inner',on='Models')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_model = pd.DataFrame( np.array(test_df1[['Sample_y','Accuracy_y', 'Accuracy with 70% best features_y','Sample_x','Accuracy_x', 'Accuracy with 70% best features_x']]),\n            index=test_df1['Models'], columns=['','Accuracy', 'Accuracy with 70% best features','','Accuracy1', 'Accuracy with 70% best features'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#final_model= pd.read_csv('models_stemmisation.csv')\nfinal_model.sort_values(by=['Accuracy1'], ascending=False, axis=0, inplace=True)\nfinal_model.rename(index=str,columns={'Unnamed: 1':'','Unnamed: 4':'', 'Accuracy':'Accuracy1', 'Accuracy with 70% best features.1':'Accuracy with 70% best features'},inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### **1.2 MODELISATION AVEC TF-IDF A PARTIR DES DONNEES SOUMISES A LA LEMMENTIZATION (1,1)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Nous utiliserons dans cette étape les modèles preselectionnés dans l'étape précedente.\ngeneralized_linear_model2 = [LogisticRegressionCV,PassiveAggressiveClassifier]\nsupport_vector_machines2 =  [LinearSVC,NuSVC]\nensemble_methods2 = [RandomForestClassifier , ExtraTreesClassifier]\nNeural_network= [MLPClassifier]\nnaive_bayes_model = [MultinomialNB, ComplementNB]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_models = pd.DataFrame({'Models':[], 'Sample':[], 'Accuracy':[],'Accuracy with 70% best features':[]})\ndata=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(generalized_linear_model,X_tfidf_Lem_vect_train,y_cora_train,X_tfidf_Lem_vect_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(support_vector_machines,X_tfidf_Lem_vect_train,y_cora_train,X_tfidf_Lem_vect_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(ensemble_methods,X_tfidf_Lem_vect_train,y_cora_train,X_tfidf_Lem_vect_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(naive_bayes_model,X_tfidf_Lem_vect_train,y_cora_train,X_tfidf_Lem_vect_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodelize(Neural_network,X_tfidf_Lem_vect_train,y_cora_train,X_tfidf_Lem_vect_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodelize([DecisionTreeClassifier, SVC],X_tfidf_Lem_vect_train,y_cora_train,X_tfidf_Lem_vect_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_df =pd.concat([pd.DataFrame(data),df_models],ignore_index=True)  \n#models_df.sort_values(by=['Accuracy','Models','Sample'], ascending=False, axis=0, inplace=True)\n#test_df = pd.DataFrame(np.array(models_df[['Accuracy', 'Accuracy with 70% best features']]),\n#                       index=[models_df['Models'],models_df['Sample']], columns=['Accuracy', 'Accuracy with 70% best features'])\ntest_df1=pd.merge(models_df[models_df['Sample']=='Test'],models_df[models_df['Sample']=='Training'],how='inner',on='Models')\ntest_df1.sort_values(by=['Accuracy_x'], ascending=False, axis=0, inplace=True)\nfinal_model2 = pd.DataFrame( np.array(test_df1[['Sample_y','Accuracy_y', 'Accuracy with 70% best features_y','Sample_x','Accuracy_x', 'Accuracy with 70% best features_x']]),\n            index=test_df1['Models'], columns=['','Accuracy', 'Accuracy with 70% best features','','Accuracy1', 'Accuracy with 70% best features'])\n\nfinal_model2.rename(index=str,columns={'Unnamed: 1':'','Unnamed: 4':'', 'Accuracy':'Accuracy1', 'Accuracy with 70% best features.1':'Accuracy with 70% best features'},inplace=True)\nfinal_model2.to_csv('models_stemmisation1_2.csv')\nfinal_model2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Conclusion : \n    Pour le modèle LinearSVC on constate que des performance identiques lorsque l'on passe des données vectorizée en stemming au données vectorizée lemmentizées. \n    Pour les autres modèles le lemmentization l'emporte sur les données en stemming.**"},{"metadata":{},"cell_type":"markdown","source":"#### **1.3 MODELISATION AVEC TF-IDF A PARTIR DE DONNEES STEMMISEES N_GRAMS(1,2)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_models = pd.DataFrame({'Models':[], 'Sample':[], 'Accuracy':[],'Accuracy with 70% best features':[]})\ndata=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(generalized_linear_model2,X_bigram_vectorizer_train,y_cora_train,X_bigram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(support_vector_machines2,X_bigram_vectorizer_train,y_cora_train,X_bigram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(ensemble_methods2,X_bigram_vectorizer_train,y_cora_train,X_bigram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(naive_bayes_model,X_bigram_vectorizer_train,y_cora_train,X_bigram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(Neural_network,X_bigram_vectorizer_train,y_cora_train,X_bigram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_df =pd.concat([pd.DataFrame(data),df_models],ignore_index=True)  \n#models_df.sort_values(by=['Accuracy','Models','Sample'], ascending=False, axis=0, inplace=True)\n#test_df = pd.DataFrame(np.array(models_df[['Accuracy', 'Accuracy with 70% best features']]),\n#                       index=[models_df['Models'],models_df['Sample']], columns=['Accuracy', 'Accuracy with 70% best features'])\ntest_df1=pd.merge(models_df[models_df['Sample']=='Test'],models_df[models_df['Sample']=='Training'],how='inner',on='Models')\ntest_df1.sort_values(by=['Accuracy_x'], ascending=False, axis=0, inplace=True)\nfinal_model2 = pd.DataFrame( np.array(test_df1[['Sample_y','Accuracy_y', 'Accuracy with 70% best features_y','Sample_x','Accuracy_x', 'Accuracy with 70% best features_x']]),\n            index=test_df1['Models'], columns=['','Accuracy', 'Accuracy with 70% best features','','Accuracy', 'Accuracy with 70% best features'])\n\nfinal_model2.rename(index=str,columns={'Unnamed: 1':'','Unnamed: 4':'', 'Accuracy.1':'Accuracy', 'Accuracy with 70% best features.1':'Accuracy with 70% best features'},inplace=True)\nfinal_model2.to_csv('models_stemmisation1_2.csv')\nfinal_model2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### **1.4 MODELISATION AVEC TF-IDF A PARTIR DES DONNEES SOUMISES A LA LEMMENTIZATION et TREE_GRAMS (1,3)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_models = pd.DataFrame({'Models':[], 'Sample':[], 'Accuracy':[],'Accuracy with 70% best features':[]})\ndata=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(generalized_linear_model2,X_Singt3gram_vectorizer_train,y_cora_train,X_Singt3gram_vectorizer_test,y_cora_test) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(support_vector_machines2,X_Singt3gram_vectorizer_train,y_cora_train,X_Singt3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(ensemble_methods2,X_Singt3gram_vectorizer_train,y_cora_train,X_Singt3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(naive_bayes_model,X_Singt3gram_vectorizer_train,y_cora_train,X_Singt3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(Neural_network,X_Singt3gram_vectorizer_train,y_cora_train,X_Singt3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_df =pd.concat([pd.DataFrame(data),df_models],ignore_index=True)  \n#models_df.sort_values(by=['Accuracy','Models','Sample'], ascending=False, axis=0, inplace=True)\n#test_df = pd.DataFrame(np.array(models_df[['Accuracy', 'Accuracy with 70% best features']]),\n#                       index=[models_df['Models'],models_df['Sample']], columns=['Accuracy', 'Accuracy with 70% best features'])\ntest_df1=pd.merge(models_df[models_df['Sample']=='Test'],models_df[models_df['Sample']=='Training'],how='inner',on='Models')\ntest_df1.sort_values(by=['Accuracy_x'], ascending=False, axis=0, inplace=True)\nfinal_model2 = pd.DataFrame( np.array(test_df1[['Sample_y','Accuracy_y', 'Accuracy with 70% best features_y','Sample_x','Accuracy_x', 'Accuracy with 70% best features_x']]),\n            index=test_df1['Models'], columns=['','Accuracy', 'Accuracy with 70% best features','','Accuracy', 'Accuracy with 70% best features'])\n\nfinal_model2.rename(index=str,columns={'Unnamed: 1':'','Unnamed: 4':'', 'Accuracy.1':'Accuracy', 'Accuracy with 70% best features.1':'Accuracy with 70% best features'},inplace=True)\nfinal_model2.to_csv('models_stemmisaton1_3.csv')\nfinal_model2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### **1.5 MODELISATION AVEC TF-IDF A PARTIR DES DONNEES SOUMISES A LA STEMMISATION et N_GRAMS (1,4)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_models = pd.DataFrame({'Models':[], 'Sample':[], 'Accuracy':[],'Accuracy with 70% best features':[]})\ndata=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(generalized_linear_model2,X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(support_vector_machines2,X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(ensemble_methods2,X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(naive_bayes_model,X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(Neural_network,X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_df =pd.concat([pd.DataFrame(data),df_models],ignore_index=True)  \n#models_df.sort_values(by=['Accuracy','Models','Sample'], ascending=False, axis=0, inplace=True)\n#test_df = pd.DataFrame(np.array(models_df[['Accuracy', 'Accuracy with 70% best features']]),\n#                       index=[models_df['Models'],models_df['Sample']], columns=['Accuracy', 'Accuracy with 70% best features'])\ntest_df1=pd.merge(models_df[models_df['Sample']=='Test'],models_df[models_df['Sample']=='Training'],how='inner',on='Models')\ntest_df1.sort_values(by=['Accuracy_x'], ascending=False, axis=0, inplace=True)\nfinal_model2 = pd.DataFrame( np.array(test_df1[['Sample_y','Accuracy_y', 'Accuracy with 70% best features_y','Sample_x','Accuracy_x', 'Accuracy with 70% best features_x']]),\n            index=test_df1['Models'], columns=['','Accuracy', 'Accuracy with 70% best features','','Accuracy', 'Accuracy with 70% best features'])\n\nfinal_model2.rename(index=str,columns={'Unnamed: 1':'','Unnamed: 4':'', 'Accuracy.1':'Accuracy', 'Accuracy with 70% best features.1':'Accuracy with 70% best features'},inplace=True)\nfinal_model2.to_csv('models_stemmisaton1_4.csv')\nfinal_model2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"On constate que la technique de lemmentization donne de meilleurs resultats pour les algorithmes des ensembles methods (random forest, extra Trees, Gradiant Boosting).\nAussi on remarque que les n_gram(1,3) donne de meilleurs resultats que la lemmetization sur les modèle lineaire de type regression logistique et Passive Agressive Classifier. Cette remarque est valable pour le linearSCV.\nPour le MultiLayer Perceptron le resultat est meilleur pour les n_gram(1,3)"},{"metadata":{},"cell_type":"markdown","source":"#### **2.1 MODELISATION AVEC Doc2Vec A PARTIR DES DONNEES SOUMISES A LA STEMMISATION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_models = pd.DataFrame({'Models':[], 'Sample':[], 'Accuracy':[],'Accuracy with 70% best features':[]})\ndata=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def modelize(list_clf,X,y,X_test,y_test):\n    selector = SelectPercentile(mutual_info_classif,percentile=70)\n    for clf in list_clf:\n        Clf1 = clf().fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf())]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=2 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted'),accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Training', 'Accuracy':accuracy_score(y,Clf1.predict(X)),'Accuracy with 70% best features':accuracy_score(y,Clf2.predict(X)) })\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Test', 'Accuracy':accuracy_score(y_test,Clf1.predict(X_test)),'Accuracy with 70% best features':accuracy_score(y_test,Clf2.predict(X_test)) })\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(generalized_linear_model2,d2v_vecs,y_cora_train,d2v_test,y_cora_test) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodelize(support_vector_machines2,d2v_vecs,y_cora_train,d2v_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(ensemble_methods2,d2v_vecs,y_cora_train,d2v_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(Neural_network,d2v_vecs,y_cora_train,d2v_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize([DecisionTreeClassifier,AdaBoostClassifier,GradientBoostingClassifier, SVC],d2v_vecs,y_cora_train,d2v_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_df =pd.concat([pd.DataFrame(data),df_models],ignore_index=True)  \n#models_df.sort_values(by=['Accuracy','Models','Sample'], ascending=False, axis=0, inplace=True)\n#test_df = pd.DataFrame(np.array(models_df[['Accuracy', 'Accuracy with 70% best features']]),\n#                       index=[models_df['Models'],models_df['Sample']], columns=['Accuracy', 'Accuracy with 70% best features'])\ntest_df1=pd.merge(models_df[models_df['Sample']=='Test'],models_df[models_df['Sample']=='Training'],how='inner',on='Models')\ntest_df1.sort_values(by=['Accuracy_x'], ascending=False, axis=0, inplace=True)\nfinal_model2 = pd.DataFrame( np.array(test_df1[['Sample_y','Accuracy_y', 'Accuracy with 70% best features_y','Sample_x','Accuracy_x', 'Accuracy with 70% best features_x']]),\n            index=test_df1['Models'], columns=['','Accuracy', 'Accuracy with 70% best features','','Accuracy', 'Accuracy with 70% best features'])\n\nfinal_model2.rename(index=str,columns={'Unnamed: 1':'','Unnamed: 4':'', 'Accuracy.1':'Accuracy', 'Accuracy with 70% best features.1':'Accuracy with 70% best features'},inplace=True)\nfinal_model2.to_csv('models_stemmisatonDo2vec.csv')\nfinal_model2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Add some ML algo from Keras and Tensorflow"},{"metadata":{},"cell_type":"markdown","source":"#### **2.1 MODELISATION AVEC Doc2Vec A PARTIR DES DONNEES SOUMISES A LA LEMMATISATION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_models = pd.DataFrame({'Models':[], 'Sample':[], 'Accuracy':[],'Accuracy with 70% best features':[]})\ndata=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def modelize(list_clf,X,y,X_test,y_test):\n    selector = SelectPercentile(mutual_info_classif,percentile=70)\n    for clf in list_clf:\n        Clf1 = clf().fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf())]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=2 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted'),accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Training', 'Accuracy':accuracy_score(y,Clf1.predict(X)),'Accuracy with 70% best features':accuracy_score(y,Clf2.predict(X)) })\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Test', 'Accuracy':accuracy_score(y_test,Clf1.predict(X_test)),'Accuracy with 70% best features':accuracy_score(y_test,Clf2.predict(X_test)) })\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(generalized_linear_model2,d2v_vecs,y_cora_train,d2v_test,y_cora_test) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(support_vector_machines2,d2v_vecs,y_cora_train,d2v_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(ensemble_methods2,d2v_vecs,y_cora_train,d2v_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(Neural_network,d2v_vecs,y_cora_train,d2v_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize([DecisionTreeClassifier,AdaBoostClassifier,GradientBoostingClassifier, SVC],d2v_vecs,y_cora_train,d2v_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_df =pd.concat([pd.DataFrame(data),df_models],ignore_index=True)  \n#models_df.sort_values(by=['Accuracy','Models','Sample'], ascending=False, axis=0, inplace=True)\n#test_df = pd.DataFrame(np.array(models_df[['Accuracy', 'Accuracy with 70% best features']]),\n#                       index=[models_df['Models'],models_df['Sample']], columns=['Accuracy', 'Accuracy with 70% best features'])\ntest_df1=pd.merge(models_df[models_df['Sample']=='Test'],models_df[models_df['Sample']=='Training'],how='inner',on='Models')\ntest_df1.sort_values(by=['Accuracy_x'], ascending=False, axis=0, inplace=True)\nfinal_model2 = pd.DataFrame( np.array(test_df1[['Sample_y','Accuracy_y', 'Accuracy with 70% best features_y','Sample_x','Accuracy_x', 'Accuracy with 70% best features_x']]),\n            index=test_df1['Models'], columns=['','Accuracy', 'Accuracy with 70% best features','','Accuracy', 'Accuracy with 70% best features'])\n\nfinal_model2.rename(index=str,columns={'Unnamed: 1':'','Unnamed: 4':'', 'Accuracy.1':'Accuracy', 'Accuracy with 70% best features.1':'Accuracy with 70% best features'},inplace=True)\nfinal_model2.to_csv('models_stemmisatonDo2vec.csv')\nfinal_model2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### **2.1 MODELISATION AVEC Doc2Vec A PARTIR DES DONNEES SOUMISES A LA LEMMATISATION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_models = pd.DataFrame({'Models':[], 'Sample':[], 'Accuracy':[],'Accuracy with 70% best features':[]})\ndata=[]\ndef modelize(list_clf,X,y,X_test,y_test):\n    selector = SelectPercentile(mutual_info_classif,percentile=70)\n    for clf in list_clf:\n        Clf1 = clf().fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf())]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=2 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted'),accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Training', 'Accuracy':accuracy_score(y,Clf1.predict(X)),'Accuracy with 70% best features':accuracy_score(y,Clf2.predict(X)) })\n        data.insert(0,{'Models':type(Clf1).__name__, 'Sample':'Test', 'Accuracy':accuracy_score(y_test,Clf1.predict(X_test)),'Accuracy with 70% best features':accuracy_score(y_test,Clf2.predict(X_test)) })\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(generalized_linear_model2,d2v_vecs_bigram,y_cora_train,d2v_test_bigram,y_cora_test) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(support_vector_machines2,d2v_vecs_bigram,y_cora_train,d2v_test_bigram,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(ensemble_methods2,d2v_vecs_bigram,y_cora_train,d2v_test_bigram,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelize(Neural_network,d2v_vecs_bigram,y_cora_train,d2v_test_bigram,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_df =pd.concat([pd.DataFrame(data),df_models],ignore_index=True)  \n#models_df.sort_values(by=['Accuracy','Models','Sample'], ascending=False, axis=0, inplace=True)\n#test_df = pd.DataFrame(np.array(models_df[['Accuracy', 'Accuracy with 70% best features']]),\n#                       index=[models_df['Models'],models_df['Sample']], columns=['Accuracy', 'Accuracy with 70% best features'])\ntest_df1=pd.merge(models_df[models_df['Sample']=='Test'],models_df[models_df['Sample']=='Training'],how='inner',on='Models')\ntest_df1.sort_values(by=['Accuracy_x'], ascending=False, axis=0, inplace=True)\nfinal_model2 = pd.DataFrame( np.array(test_df1[['Sample_y','Accuracy_y', 'Accuracy with 70% best features_y','Sample_x','Accuracy_x', 'Accuracy with 70% best features_x']]),\n            index=test_df1['Models'], columns=['','Accuracy', 'Accuracy with 70% best features','','Accuracy', 'Accuracy with 70% best features'])\n\nfinal_model2.rename(index=str,columns={'Unnamed: 1':'','Unnamed: 4':'', 'Accuracy.1':'Accuracy', 'Accuracy with 70% best features.1':'Accuracy with 70% best features'},inplace=True)\nfinal_model2.to_csv('models_lemmATISatIonDo2vec.csv')\nfinal_model2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### HYPERPARAMETERS TUNING\nNos meilleurs modèles sont les suivants:\n    - LinearSVC appliqué avec une matrice Tf-Idf de données stemmisées en n_grams(1,3)\n    - LogisticRegression avec une matrice Tf-Idf de données stemmisées en n_grams(1,4)\n    - ExtraTreeClassifier avec une matrice Tf-Idf de données stemmisées en n_grams(1,4)\n    - MLPClassifier vec une matrice Tf-Idf de données stemmisées en n_grams(1,4)\n    \nNous appliquerons deux méthodes d'optimisation d'hyperparametre afin d'en comparer les resultats : GridSearchCV et HyperOpt    \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import cross_val_score, StratifiedShuffleSplit, KFold\nfrom sklearn.feature_selection import SelectPercentile\nfrom hyperopt import hp,fmin,Trials, tpe, STATUS_FAIL, STATUS_OK, space_eval, anneal\nfrom hyperopt.pyll import scope\nfrom hyperopt.pyll import stochastic\n\nfrom sklearn.metrics import accuracy_score, f1_score, balanced_accuracy_score\nfrom sklearn.decomposition import SparsePCA\nfrom sklearn.preprocessing import normalize, StandardScaler\n\nimport time\nrandom_state = 42\nkf = KFold(n_splits=2,random_state=random_state)\nn_iter= 50\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**1. ExtraTreeClassifier Hyperparameters Tuning**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Parameters Optimization with HyperOpt \n#Tree-structure Parzen Estimator: TPE is a default algorithm for the Hyperopt. It uses Bayesian approach for optimization. \n#At every step it is trying to build probabilistic model of the function and choose the most promising parameters for the next step.\n#1. We need to create a function to minimize.\ndef extraTree_accuracy_cv(params, random_state=random_state, cv=kf, X=X_t3gram_vectorizer_train, y=y_cora_train):\n    # the function gets a set of variable parameters in \"param\"\n    # the function gets a set of variable parameters in \"param\"\n    params = {'n_estimators': int(params['n_estimators']),  #The number of trees in the forest.\n              'max_features': str(params['max_features']), #The number of features to consider when looking for the best split.\n              'min_samples_split': int(params['min_samples_split']), #The minimum number of samples required to split an internal node\n              'max_depth': int(params['max_depth'])} #The maximum depth of the tree\n    # we use this params to create a new LinearSVC Classifier\n    model = ExtraTreesClassifier(random_state=random_state, **params, n_jobs = -1)\n    # and then conduct the cross validation with the same folds as before\n    try:\n        return {'loss' : -cross_val_score(model, X, y, cv=cv, scoring=\"accuracy\", n_jobs=-1).mean(),\n                'time' : time.time(),\n                'status' : STATUS_OK }\n    except (Exception, e):\n        return {'status' : STATUS_FAIL,\n                'time' : time.time(),\n                'exception' : str(e)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# possible values of parameters\nspace={'n_estimators': hp.quniform('n_estimators', 100, 2000, 1),\n       'max_depth' : hp.quniform('max_depth', 2, 80, 1),\n       'max_features': hp.choice('max_features', [\"sqrt\",\"log2\"]),\n       'min_samples_split': hp.quniform('min_samples_split', 2, 10, 1)  \n      }\n\n# trials will contain logging information\ntrials = Trials()\n\nbest=fmin(fn=extraTree_accuracy_cv, # function to optimize\n          space=space, \n          algo=tpe.suggest, # optimization algorithm, hyperotp will select its parameters automatically\n          max_evals=n_iter, # maximum number of iterations\n          trials=trials, # logging\n          rstate=np.random.RandomState(random_state) # fixing random state for the reproducibility\n         )\n# computing the score on the test set\nmodel = ExtraTreesClassifier(random_state=random_state, \n                             n_estimators=int(best['n_estimators']), \n                             max_depth=int(best['max_depth']),\n                             min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']),\n                             n_jobs=-1)\nmodel.fit(X_t3gram_vectorizer_train,y_cora_train)\ntpe_test_score=accuracy_score(y_cora_test, model.predict(X_t3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(stochastic.sample(space))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy score on train set {:.3f} params {}\".format( -extraTree_accuracy_cv(space_eval(space,best))['loss'], space_eval(space,best)))\nprint('Accuracy score on validation sample {:.3f}'.format(tpe_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy score on train set {:.3f} params {}\".format( -extraTree_accuracy_cv(space_eval(space,best))['loss'], space_eval(space,best)))\nprint('Accuracy score on validation sample {:.3f}'.format(tpe_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#extraTree_accuracy_cv(best), \nspace_eval(space,best)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tpe_results=np.array([[x['result']['loss'],\n                      x['misc']['vals']['max_depth'][0],\n                      x['misc']['vals']['n_estimators'][0],\n                      x['misc']['vals']['max_features'][0],\n                      x['misc']['vals']['min_samples_split'][0]\n                      \n                      \n                      ] for x in trials.trials])\n\ntpe_results_df=pd.DataFrame(tpe_results,\n                           columns=['score', 'max_depth', 'n_estimators', 'max_features', 'min_samples_split'])\ntpe_results_df.plot(subplots=True,figsize=(10, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(random_state=random_state, n_estimators=int(best['n_estimators']), max_depth=int(best['max_depth']), min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']), n_jobs=-1).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(random_state=random_state, n_estimators=int(best['n_estimators']), max_depth=int(best['max_depth']), min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']), n_jobs=-1))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=random_state)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\n    \nmodelize([ExtraTreesClassifier],X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Parameters Optimization with HyperOpt \n#Tree-structure Parzen Estimator: TPE is a default algorithm for the Hyperopt. It uses Bayesian approach for optimization. \n#At every step it is trying to build probabilistic model of the function and choose the most promising parameters for the next step.\n#1. We need to create a function to minimize.\ndef RF_cv(params, random_state=random_state, cv=kf, X=X_t3gram_vectorizer_train, y=y_cora_train):\n    # the function gets a set of variable parameters in \"param\"\n    # the function gets a set of variable parameters in \"param\"\n    params = {'n_estimators': int(params['n_estimators']),  #The number of trees in the forest.\n              'max_features': str(params['max_features']), #The number of features to consider when looking for the best split.\n              'min_samples_split': int(params['min_samples_split']), #The minimum number of samples required to split an internal node\n              'max_depth': int(params['max_depth'])} #The maximum depth of the tree\n    # we use this params to create a new LinearSVC Classifier\n    model = RandomForestClassifier(random_state=random_state, **params, n_jobs = -1)\n    # and then conduct the cross validation with the same folds as before\n    try:\n        return {'loss' : -cross_val_score(model, X, y, cv=cv, scoring=\"accuracy\", n_jobs=-1).mean(),\n                'time' : time.time(),\n                'status' : STATUS_OK }\n    except (Exception, e):\n        return {'status' : STATUS_FAIL,\n                'time' : time.time(),\n                'exception' : str(e)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# possible values of parameters\nspace={'n_estimators': hp.quniform('n_estimators', 20, 500, 1),\n       'max_depth' : hp.quniform('max_depth', 2, 100, 1),\n       'max_features': hp.choice('max_features', [\"sqrt\",\"log2\"]),\n       'min_samples_split': hp.quniform('min_samples_split', 2, 10, 1)  \n      }\n\n# trials will contain logging information\ntrials = Trials()\n\nbest=fmin(fn=RF_cv, # function to optimize\n          space=space, \n          algo=anneal.suggest, # optimization algorithm, hyperotp will select its parameters automatically\n          max_evals=n_iter, # maximum number of iterations\n          trials=trials, # logging\n          rstate=np.random.RandomState(random_state) # fixing random state for the reproducibility\n         )\n# computing the score on the test set\nmodel = RandomForestClassifier(random_state=random_state, \n                             n_estimators=int(best['n_estimators']),\n                             #n_estimators= 302,  \n                             max_depth=int(best['max_depth']),\n                             min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']),\n                             n_jobs=-1)\nmodel.fit(X_t3gram_vectorizer_train,y_cora_train)\ntpe_test_score=accuracy_score(y_cora_test, model.predict(X_t3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hp.quniform('n_estimators', 20, 500, 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy score on train set {:.3f} params {}\".format( -RF_cv(space_eval(space,best))['loss'], space_eval(space,best)))\nprint('Accuracy score on validation sample {:.3f}'.format(tpe_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tpe_results=np.array([[x['result']['loss'],\n                      x['misc']['vals']['max_depth'][0],\n                      #x['misc']['vals']['n_estimators'][0],\n                     # x['misc']['vals']['max_features'][0],\n                      x['misc']['vals']['min_samples_split'][0]\n                      \n                      \n                      ] for x in trials.trials])\n\ntpe_results_df=pd.DataFrame(tpe_results,\n                           columns=['score', 'max_depth', #'n_estimators',# 'max_features', \n                                    'min_samples_split'])\ntpe_results_df.plot(subplots=True,figsize=(10, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        #Clf1 = clf(random_state=random_state, n_estimators=int(best['n_estimators']), max_depth=int(best['max_depth']), min_samples_split= int(space_eval(space,best)['min_samples_split']),\n        #                     max_features=str(space_eval(space,best)['max_features']), n_jobs=-1).fit(X,y)\n        #Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(random_state=random_state, n_estimators=int(best['n_estimators']), max_depth=int(best['max_depth']), min_samples_split= int(space_eval(space,best)['min_samples_split']),\n        #                     max_features=str(space_eval(space,best)['max_features']), n_jobs=-1))]).fit(X,y)\n        Clf1 = clf(random_state=random_state, n_estimators=302, max_depth=96, n_jobs=-1).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(random_state=random_state, n_estimators=302, max_depth=96,n_jobs=-1) )]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=random_state)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\n    \nmodelize([RandomForestClassifier],X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import validation_curve\nfrom matplotlib import pyplot as plt\n\ndef plot_validation_curve(estimator, X, y, param_name, param_range,\n                          ylim=(0, 1.1), cv=5, n_jobs=-1, scoring=None):\n    estimator_name = type(estimator).__name__\n    plt.figure(figsize=(10,6))\n    plt.title(\"Validation curves for %s on %s\"\n              % (param_name, estimator_name))\n    plt.grid()\n    plt.xlim(min(param_range), max(param_range))\n    plt.xlabel(param_name)\n    plt.ylabel(\"Score\")\n    plt.xscale('log')\n    \n    train_scores, test_scores = validation_curve(\n        estimator, X, y, param_name, param_range,\n        cv=cv, n_jobs=n_jobs, scoring=scoring)\n\n    train_scores_mean = np.mean(train_scores, axis=1)\n    test_scores_mean = np.mean(test_scores, axis=1)\n    train_scores_std = np.std(train_scores, axis=1)\n    test_scores_std = np.std(test_scores, axis=1)\n    \n    plt.plot(\n        param_range, train_scores_mean, 'o-',\n        color=\"firebrick\", linewidth=3, \n        label=\"Training score\")\n    plt.plot(\n        param_range, test_scores_mean, 'o-',\n        color=\"darkgoldenrod\", \n        label=\"Cross-validation score\")\n    plt.fill_between(\n        param_range, train_scores_mean - train_scores_std,\n        train_scores_mean + train_scores_std, alpha=0.1,\n        color=\"firebrick\")\n    plt.fill_between(\n        param_range, test_scores_mean - test_scores_std,\n        test_scores_mean + test_scores_std, alpha=0.1, color=\"darkgoldenrod\")\n    plt.legend(loc=\"best\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n%matplotlib inline\ncv = StratifiedShuffleSplit(n_splits=10 #nombre de fois qu'on regenere le train test\n                            , train_size=.8, random_state=random_state)\n\nclf = ExtraTreesClassifier(n_jobs=-1)\nparam_name = 'max_depth'\nparam_range = np.linspace(1,1500,15,dtype=int)\nprint(param_range)\nplot_validation_curve(\n    clf, X_t3gram_vectorizer_train, y_cora_train, param_name, param_range, cv=cv.split(X_t3gram_vectorizer_train, y_cora_train), scoring='accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param_range","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n%matplotlib inline\ncv = StratifiedShuffleSplit(n_splits=1 #nombre de fois qu'on regenere le train test\n                            , train_size=.7, random_state=random_state)\n\nclf = ExtraTreesClassifier(n_jobs=-1)\nparam_name = 'n_estimators'\nparam_range = np.linspace(1,500,10,dtype=int)\nprint(param_range)\nplot_validation_curve(\n    clf, X_t3gram_vectorizer_train, y_cora_train, param_name, param_range, cv=cv.split(X_t3gram_vectorizer_train, y_cora_train), scoring='accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n#Parameters Optimization with HyperOpt \n#Tree-structure Parzen Estimator: TPE is a default algorithm for the Hyperopt. It uses Bayesian approach for optimization. \n#At every step it is trying to build probabilistic model of the function and choose the most promising parameters for the next step.\n#1. We need to create a function to minimize.\ndef ExtraTC_cv(params, random_state=random_state, cv=kf, X=X_t3gram_vectorizer_train, y=y_cora_train):\n    # the function gets a set of variable parameters in \"param\"\n    # the function gets a set of variable parameters in \"param\"\n    params = {#'n_estimators': int(params['n_estimators']),  #The number of trees in the forest.\n              'max_features': str(params['max_features']), #The number of features to consider when looking for the best split.\n              'min_samples_split': int(params['min_samples_split']), #The minimum number of samples required to split an internal node\n              'max_depth': int(params['max_depth'])} #The maximum depth of the tree\n    # we use this params to create a new LinearSVC Classifier\n    model = ExtraTreesClassifier(random_state=random_state, **params, n_jobs = -1, n_estimators=110)\n    # and then conduct the cross validation with the same folds as before\n    try:\n        return {'loss' : -cross_val_score(model, X, y, cv=cv, scoring=\"accuracy\", n_jobs=-1).mean(),\n                'time' : time.time(),\n                'status' : STATUS_OK }\n    except (Exception, e):\n        return {'status' : STATUS_FAIL,\n                'time' : time.time(),\n                'exception' : str(e)}\n    \n# possible values of parameters\nspace={#'n_estimators': hp.quniform('n_estimators', 80, 180, 1),\n       'max_depth' : hp.quniform('max_depth', 215, 430, 1),\n       'max_features': hp.choice('max_features', [\"sqrt\",\"log2\"]),\n       'min_samples_split': hp.quniform('min_samples_split', 2, 10, 1)  \n      }\n\n# trials will contain logging information\ntrials = Trials()\n\nbest=fmin(fn=ExtraTC_cv, # function to optimize\n          space=space, \n          algo=anneal.suggest, # optimization algorithm, hyperotp will select its parameters automatically\n          max_evals=n_iter, # maximum number of iterations\n          trials=trials, # logging\n          rstate=np.random.RandomState(random_state) # fixing random state for the reproducibility\n         )\n# computing the score on the test set\nmodel = ExtraTreesClassifier(random_state=random_state, \n                             #n_estimators=int(best['n_estimators']),\n                             n_estimators= 110,  \n                             max_depth=int(best['max_depth']),\n                             min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']),\n                             n_jobs=-1)\nmodel.fit(X_t3gram_vectorizer_train,y_cora_train)\ntpe_test_score=accuracy_score(y_cora_test, model.predict(X_t3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy score on train set {:.3f} params {}\".format( -ExtraTC_cv(space_eval(space,best))['loss'], space_eval(space,best)))\nprint('Accuracy score on validation sample {:.3f}'.format(tpe_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(random_state=random_state, n_estimators=110, max_depth=int(best['max_depth']), min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']), n_jobs=-1).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(random_state=random_state, n_estimators=110, max_depth=int(best['max_depth']), min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']), n_jobs=-1))]).fit(X,y)\n        #Clf1 = clf(random_state=random_state, n_estimators=302, max_depth=96, n_jobs=-1).fit(X,y)\n        #Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(random_state=random_state, n_estimators=302, max_depth=96,n_jobs=-1) )]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=random_state)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\n    \nmodelize([ExtraTreesClassifier],X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n%matplotlib inline\ncv = StratifiedShuffleSplit(n_splits=1 #nombre de fois qu'on regenere le train test\n                            , train_size=.7, random_state=random_state)\n\nclf = ExtraTreesClassifier(n_jobs=-1)\nparam_name = 'max_depth'\nparam_range = np.linspace(1,1500,15,dtype=int)\nprint(param_range)\nplot_validation_curve(\n    clf, X_t3gram_vectorizer_train_, y_cora_train, param_name, param_range, cv=cv.split(X_t3gram_vectorizer_train_, y_cora_train), scoring='accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n%matplotlib inline\ncv = StratifiedShuffleSplit(n_splits=1 #nombre de fois qu'on regenere le train test\n                            , train_size=.7, random_state=random_state)\n\nclf = ExtraTreesClassifier(n_jobs=-1)\nparam_name = 'n_estimators'\nparam_range = np.linspace(1,700,7,dtype=int)\nprint(param_range)\nplot_validation_curve(\n    clf, X_t3gram_vectorizer_train, y_cora_train, param_name, param_range, cv=cv.split(X_t3gram_vectorizer_train, y_cora_train), scoring='accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndef ExtraTC_cv(params, random_state=random_state, cv=kf, X=X_t3gram_vectorizer_train, y=y_cora_train):\n    # the function gets a set of variable parameters in \"param\"\n    # the function gets a set of variable parameters in \"param\"\n    params = {#'n_estimators': int(params['n_estimators']),  #The number of trees in the forest.\n              'max_features': str(params['max_features']), #The number of features to consider when looking for the best split.\n              'min_samples_split': int(params['min_samples_split']), #The minimum number of samples required to split an internal node\n              'max_depth': int(params['max_depth'])} #The maximum depth of the tree\n    # we use this params to create a new LinearSVC Classifier\n    model = ExtraTreesClassifier(random_state=random_state, **params, n_jobs = -1, n_estimators=110)\n    # and then conduct the cross validation with the same folds as before\n    return {'loss' : -cross_val_score(model, X, y, cv=cv, scoring=\"accuracy\", n_jobs=-1).mean(),\n                'time' : time.time(),\n                'status' : STATUS_OK }\n\n    \n# possible values of parameters\nspace={#'n_estimators': hp.quniform('n_estimators', 80, 180, 1),\n       'max_depth' : hp.quniform('max_depth', 400, 500, 1),\n       'max_features': hp.choice('max_features', [\"sqrt\",\"log2\"]),\n       'min_samples_split': hp.quniform('min_samples_split', 2, 10, 1)  \n      }\n\n# trials will contain logging information\ntrials = Trials()\n\nbest=fmin(fn=ExtraTC_cv, # function to optimize\n          space=space, \n          algo=anneal.suggest, # optimization algorithm, hyperotp will select its parameters automatically\n          max_evals=n_iter, # maximum number of iterations\n          trials=trials, # logging\n          rstate=np.random.RandomState(random_state) # fixing random state for the reproducibility\n         )\n# computing the score on the test set\nmodel = ExtraTreesClassifier(random_state=random_state, \n                             #n_estimators=int(best['n_estimators']),\n                             n_estimators= 117,  \n                             max_depth=int(best['max_depth']),\n                             min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']),\n                             n_jobs=-1)\nmodel.fit(X_t3gram_vectorizer_train_,y_cora_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy score on train set {:.3f} params {}\".format( -ExtraTC_cv(space_eval(space,best))['loss'], space_eval(space,best)))\ntpe_test_score=accuracy_score(y_cora_test, model.predict(X_t3gram_vectorizer_test))\nprint('Accuracy score on validation sample {:.3f}'.format(tpe_test_score))\ntpe_test_score=balanced_accuracy_score(y_cora_test, model.predict(X_t3gram_vectorizer_test))\nprint('balanced Accuracy score on validation sample {:.3f}'.format(tpe_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(random_state=random_state, n_estimators=117, max_depth=int(best['max_depth']), min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']), n_jobs=-1).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(random_state=random_state, n_estimators=110, max_depth=int(best['max_depth']), min_samples_split= int(space_eval(space,best)['min_samples_split']),\n                             max_features=str(space_eval(space,best)['max_features']), n_jobs=-1))]).fit(X,y)\n        #Clf1 = clf(random_state=random_state, n_estimators=302, max_depth=96, n_jobs=-1).fit(X,y)\n        #Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(random_state=random_state, n_estimators=302, max_depth=96,n_jobs=-1) )]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=random_state)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Balanced Accuracy Score    F1 Score                Balanced Accuracy Score    F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),\n                balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted'),\n                balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\n    \nmodelize([ExtraTreesClassifier],X_t3gram_vectorizer_train_,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#OPTIMISATION DES HYPERPARAMETRES DU MODEL LINEARSVC AVEC GRIDSEARCH - We want to decrease the classifier's complexty (decrease parameter C of LinearSVC)\nfrom sklearn.model_selection import GridSearchCV\nparam_grid={'max_depth': np.linspace( 10, 200, 10,dtype=int), # Maximum number of levels in tree\n            'n_estimators': np.linspace(100,1000, 10 ,dtype=int), # Number of trees in random forest\n            'max_features' : ['auto', 'log2'], # Number of features to consider at every split\n            'min_samples_split' : [2, 5, 10], # Minimum number of samples required to split a node\n            #'min_samples_leaf': [1, 2, 4], # Minimum number of samples required at each leaf node\n           }\nmodel = RandomForestClassifier(random_state=random_state, n_jobs=-1)\nkf = KFold(n_splits=3,random_state=random_state)\n\ngs=GridSearchCV(model, param_grid, scoring='accuracy', n_jobs=-1, cv=kf, verbose=False)\ngs.fit(X_t3gram_vectorizer_train,y_cora_train)\ngs_test_score=accuracy_score(y_cora_test, gs.predict(X_t3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gs_results_df=pd.DataFrame(np.transpose([gs.cv_results_['mean_test_score'],\n                                         gs.cv_results_['param_max_depth'].data,\n                                         gs.cv_results_['param_max_features'].data,\n                                         gs.cv_results_['param_min_samples_split'].data,\n                                         gs.cv_results_['param_n_estimators'].data]),\n                                           \n                           columns=['score', 'max_depth', 'max_features', 'min_samples_split', 'n_estimators',])\ngs_results_df.plot(subplots=True,figsize=(10, 10))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print((gs.cv_results_).keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gs_results_df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#gs_results_df.plot([gs_results_df['max_depth'],gs_results_df['score']])\nplt.plot(gs_results_df['max_depth'],gs_results_df['score'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**2. LinearSVC Hyperparameters Tuning**"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#OPTIMISATION DES HYPERPARAMETRES DU MODEL LINEARSVC AVEC GRIDSEARCH - We want to decrease the classifier's complexty (decrease parameter C of LinearSVC)\nfrom sklearn.model_selection import GridSearchCV\nparam_grid={'C': np.linspace(0.0000001, 1, 250)}\nmodel = LinearSVC()\nkf = KFold(n_splits=2,random_state=42)\nn_iter= 50\nrandom_state = 42\ngs=GridSearchCV(model, param_grid, scoring='accuracy', n_jobs=-1, cv=kf, verbose=False)\ngs.fit(X_Singt3gram_vectorizer_train,y_cora_train)\ngs_test_score=accuracy_score(y_cora_test, gs.predict(X_Singt3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy for training{:.3f} params {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy for validation: {:.3f}\".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gs_results_df=pd.DataFrame(np.transpose([gs.cv_results_['mean_test_score'],\n                                         gs.cv_results_['param_C'].data]),\n                           columns=['score', 'c'])\ngs_results_df.plot(subplots=True,figsize=(10, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Let's RECALL THE lINEARsvc with the penality = l1, which results in sparse solutions. Sparse solutions correspond to an implicit feature selection.\nfrom sklearn.model_selection import GridSearchCV\nparam_grid={'C': np.linspace(0.0000001, 1, 250),\n            'penalty' : ['l1'],\n            'dual': [False]}\nmodel = LinearSVC()\nkf = KFold(n_splits=2,random_state=42)\nn_iter= 50\nrandom_state = 42\ngs=GridSearchCV(model, param_grid, scoring='accuracy', n_jobs=-1, cv=kf, verbose=False)\ngs.fit(X_Singt3gram_vectorizer_train,y_cora_train)\ngs_test_score=accuracy_score(y_cora_test, gs.predict(X_Singt3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gs_results_df=pd.DataFrame(np.transpose([gs.cv_results_['mean_test_score'],\n                                         gs.cv_results_['param_C'].data]),\n                           columns=['score', 'c'])\ngs_results_df.plot(subplots=True,figsize=(10, 10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**We prefer the hyperparameter in the first case: C = 0.46231161155778894 with best accuracy score = 0.878** "},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize_LinearSVC(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(C=0.46231161155778894).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(C=0.46231161155778894))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=42)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\n    \nmodelize_LinearSVC([LinearSVC],X_Singt3gram_vectorizer_train,y_cora_train,X_Singt3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Nouvel Echantillonage  \ncora_data_neg_sample = cora_data[cora_data['target'] == 1] #Negatives comments\ncora_data_positive_sample = cora_data[cora_data['target'] == 0].reindex()  #Positive Comments\n\ncora_resampling = pd.concat([pd.DataFrame(cora_data_positive_sample.sample(20000)),\n                               pd.DataFrame(cora_data_neg_sample)])\n100*(cora_resampling.groupby('target')['question_text'].count())/cora_resampling['target'].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cora_resampling.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Cleaning the data \ncora_resampling['clean_question'] = cora_resampling['question_text'].apply(clean_str)\n#Tokenizing and stopwords removing\ncora_resampling['tokeniZ_stopWords_question'] = cora_resampling['clean_question'].apply(tokeniZ_stopWords)\n#Words Stemming\ncora_resampling['stemming_question'] = [[ps.stem(word) for word in words] for words in cora_resampling['tokeniZ_stopWords_question'] ]\ncora_resampling['stemming_question_for_tfidf'] = [' '.join(words) for words in cora_resampling['stemming_question']] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cora_resampling.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_cora_train, X_cora_test, y_cora_train, y_cora_test = train_test_split(\n            cora_resampling['stemming_question_for_tfidf']\n            ,cora_resampling['target'], \n            test_size=0.3, random_state=42)\nX_cora_train.shape, X_cora_test.shape, y_cora_train.shape, y_cora_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Vectorization with tf-idf\nX_Singt3gram_vectorizer_train_ = st3gram_vectorizer.fit_transform(X_cora_train_)\nX_Singt3gram_vectorizer_test_  = st3gram_vectorizer.transform(X_cora_test_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#OPTIMISATION DES HYPERPARAMETRES DU MODEL LINEARSVC AVEC GRIDSEARCH - We want to increase the regularization of the classifier (decrease parameter C of LinearSVC)\nfrom sklearn.model_selection import GridSearchCV\nparam_grid={'C': np.linspace(0.0000001, 1, 250)}\nmodel = LinearSVC()\nkf = KFold(n_splits=2,random_state=42)\nn_iter= 50\nrandom_state = 42\ngs=GridSearchCV(model, param_grid, scoring='accuracy', n_jobs=-1, cv=kf, verbose=False)\ngs.fit(X_Singt3gram_vectorizer_train_,y_cora_train_)\ngs_test_score=accuracy_score(y_cora_test_, gs.predict(X_Singt3gram_vectorizer_test_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import balanced_accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Let's try to increase the data regularization by using the Standardization method\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=60)\ndef modelize_LinearSVC(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(C=0.4056225493975904).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(C=0.4056225493975904))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Balanced Accuracy Score    F1 Score                Balanced Accuracy Score    F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='balanced_accuracy')\n        print('=============================================')\n    \nmodelize_LinearSVC([LinearSVC],X_Singt3gram_vectorizer_train,y_cora_train,X_Singt3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Let's try to increase the data regularization by using the Standardization method\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=60)\ndef modelize_LinearSVC(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(C=0.4056225493975904).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(C=0.4056225493975904))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Balanced Accuracy Score    F1 Score                Balanced Accuracy Score    F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='balanced_accuracy')\n        print('=============================================')\nnorm=StandardScaler(with_mean=False)\nmodelize_LinearSVC([LinearSVC],norm.fit_transform(X_Singt3gram_vectorizer_train),y_cora_train,norm.transform(X_Singt3gram_vectorizer_test),y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Let's try to increase the data regularization by using the normalization method\nmodelize_LinearSVC([LinearSVC],normalize(X_Singt3gram_vectorizer_train,norm='l2'),y_cora_train,normalize(X_Singt3gram_vectorizer_test,norm='l2'),y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**3. MLPClassiffer Hyperparameters optimization**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Parameters Optimization with HyperOpt \n#1. We need to create a function to minimize.\ndef MLP_accuracy_cv(params, random_state=random_state, cv=kf, X=X_t3gram_vectorizer_train, y=y_cora_train):\n    # the function gets a set of variable parameters in \"param\" \n    params = {'hidden_layer_sizes': tuple(params['hidden_layer_sizes']), \n              'activation': str(params['activation']),#Activation functions for the hidden layers\n              'solver': str(params['solver']), #The solver for weight optimization.\n              'alpha': int(params['alpha']), #L2 penalty (regularization term) parameter\n              'learning_rate': str(params['learning_rate'])} #Learning rate schedule for weight updates\n    # we use this params to create a new LinearSVC Classifier\n    model = MLPClassifier(random_state=random_state ,# **params)\n                          hidden_layer_sizes = params['hidden_layer_sizes'],\n                          activation = params['activation'],\n                          solver = params['solver'],\n                          alpha = params['alpha'],\n                          learning_rate = params['learning_rate'])\n    # and then conduct the cross validation with the same folds as before\n    try:\n        return {'loss' : -cross_val_score(model, X, y, cv=cv, scoring=\"accuracy\", n_jobs=-1).mean(),\n                'time' : time.time(),\n                'status' : STATUS_OK }\n    except (Exception, e):\n        return {'status' : STATUS_FAIL,\n                'time' : time.time(),\n                'exception' : str(e)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# possible values of parameters\nspace={'hidden_layer_sizes': hp.choice('hidden_layer_sizes' , [(20,10,5,),(5,25,50,),(100,25,5,)]),\n       'activation' : hp.choice('activation' , [\"identity\", \"logistic\", \"tanh\", \"relu\"]), \n       'solver' : hp.choice( 'solver' , [\"lbfgs\", \"sgd\", \"adam\"]),\n       'alpha': hp.uniform('alpha',0.0001,0.9),\n       'learning_rate': hp.choice('learning_rate' , [\"constant\", \"invscaling\", \"adaptive\"])\n      }\n      \n\n# trials will contain logging information\ntrials = Trials()\n\nbest=fmin(fn=MLP_accuracy_cv, # function to optimize\n          space=space, \n          algo=tpe.suggest, # optimization algorithm, hyperotp will select its parameters automatically\n          max_evals=n_iter, # maximum number of iterations\n          trials=trials, # logging\n          rstate=np.random.RandomState(random_state)) # fixing random state for the reproducibility","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# computing the score on the test set\nmodel = MLPClassifier(random_state=random_state, activation = str(space_eval(space,best)['activation'] ),\n                                                 solver = str(space_eval(space,best)['solver']),\n                                                 alpha = int(best['alpha']),\n                                                 hidden_layer_sizes=tuple(space_eval(space,best)['hidden_layer_sizes']),\n                                                 learning_rate = str(space_eval(space,best)['learning_rate'] ))\nmodel.fit(X_t3gram_vectorizer_train,y_cora_train)\ntpe_test_score=accuracy_score(y_cora_test, model.predict(X_t3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprint(\"Best Accuracy score on train set {:.3f} params {}\".format( -MLP_accuracy_cv(space_eval(space,best))['loss'], space_eval(space,best)))\nprint('Accuracy score on validation sample {:.3f}'.format(tpe_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize_MLP(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Model = MLPClassifier(random_state=random_state, activation = str(space_eval(space,best)['activation'] ),\n                                                 solver = str(space_eval(space,best)['solver']),\n                                                 alpha = int(best['alpha']),\n                                                 hidden_layer_sizes=tuple(space_eval(space,best)['hidden_layer_sizes']),\n                                                 learning_rate = str(space_eval(space,best)['learning_rate'] ))\n        \n        Clf1 = Model.fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',MLPClassifier(random_state=random_state, activation = str(space_eval(space,best)['activation'] ),\n                                                 solver = str(space_eval(space,best)['solver']),\n                                                 alpha = int(best['alpha']),\n                                                 hidden_layer_sizes=tuple(space_eval(space,best)['hidden_layer_sizes']),\n                                                 learning_rate = str(space_eval(space,best)['learning_rate'] )))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=random_state)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\n        \nmodelize_MLP([MLPClassifier],X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cora_data_neg_sample = cora_data[cora_data['target'] == 1] #Negatives comments\ncora_data_positive_sample = cora_data[cora_data['target'] == 0].reindex()  #Positive Comments\n\ncora_resampling = pd.concat([pd.DataFrame(cora_data_positive_sample.sample(4000)),\n                               pd.DataFrame(cora_data_neg_sample.sample(2500))])\n100*(cora_resampling.groupby('target')['question_text'].count())/cora_resampling['target'].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Cleaning the data \ncora_resampling['clean_question'] = cora_resampling['question_text'].apply(clean_str)\n#Tokenizing and stopwords removing\ncora_resampling['tokeniZ_stopWords_question'] = cora_resampling['clean_question'].apply(tokeniZ_stopWords)\n#Words lemmatization\ncora_resampling['lemmatize_question'] = cora_resampling['tokeniZ_stopWords_question'].apply(lemat_words)\ncora_resampling['lemmatize_question_for_tfidf'] = [' '.join(x) for x in cora_resampling['lemmatize_question'] ]\n#Vectorization with tf-idf\nX_Singt3gram_vectorizer_train = st3gram_vectorizer.fit_transform(X_cora_train['lemmatize_question_for_tfidf'])\nX_Singt3gram_vectorizer_test  = st3gram_vectorizer.transform(X_cora_test['lemmatize_question_for_tfidf'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize_MLP(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Model = MLPClassifier(random_state=random_state, activation = str(space_eval(space,best)['activation'] ),\n                                                 solver = str(space_eval(space,best)['solver']),\n                                                 alpha = int(best['alpha']),\n                                                 hidden_layer_sizes=tuple(space_eval(space,best)['hidden_layer_sizes']),\n                                                 learning_rate = str(space_eval(space,best)['learning_rate'] ))\n        \n        Clf1 = Model.fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',MLPClassifier(random_state=random_state, activation = str(space_eval(space,best)['activation'] ),\n                                                 solver = str(space_eval(space,best)['solver']),\n                                                 alpha = int(best['alpha']),\n                                                 hidden_layer_sizes=tuple(space_eval(space,best)['hidden_layer_sizes']),\n                                                 learning_rate = str(space_eval(space,best)['learning_rate'] )))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=random_state)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Balanced Accuracy Score    F1 Score                Balanced Accuracy Score    F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='balanced_accuracy')\n        print('=============================================')\n        \nmodelize_MLP([MLPClassifier],X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**4. Logistic Regression Hyperparameters tuning**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GridSearchCV, KFold\nrandom_state = 42","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Let's try to increase the model complexity.\n\nparam_grid={'C': np.linspace(1, 12, 100),\n            'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga', 'liblinear']}\nmodel = LogisticRegression( n_jobs=-1)\nkf = KFold(n_splits=3,random_state=random_state)\n\ngs=GridSearchCV(model, param_grid, scoring='accuracy',  n_jobs=-1, cv=kf, verbose=False)\ngs.fit(X_t3gram_vectorizer_train,y_cora_train)\ngs_test_score=accuracy_score(y_cora_test, gs.predict(X_t3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gs_results_df=pd.DataFrame(np.transpose([gs.cv_results_['mean_test_score'],\n                                         gs.cv_results_['param_C'].data]),\n                           columns=['score', 'c'])\ngs_results_df.plot(subplots=True,figsize=(10, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=71)\ndef modelize_LogReg(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(C=3.3855421686746987).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(C=3.3855421686746987))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=42)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),\n                balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted'),\n                balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\n    \nmodelize_LogReg([LogisticRegression],X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Let's try to increase the model complexity.\n\nparam_grid={'C': np.linspace(1, 100, 250), 'penalty': ['l1']}\nmodel = LogisticRegression()\nkf = KFold(n_splits=3,random_state=random_state)\n\ngs=GridSearchCV(model, param_grid, scoring='accuracy',  n_jobs=-1, cv=kf, verbose=False)\ngs.fit(X_t3gram_vectorizer_train,y_cora_train)\ngs_test_score=accuracy_score(y_cora_test, gs.predict(X_t3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gs_results_df=pd.DataFrame(np.transpose([gs.cv_results_['mean_test_score'],\n                                         gs.cv_results_['param_C'].data]),\n                           columns=['score', 'c'])\ngs_results_df.plot(subplots=True,figsize=(10, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize_LogReg(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(C= 2.9879518072289155, penalty= 'l1').fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(C=2.9879518072289155, penalty='l1'))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=42)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),\n                balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\nnorm=StandardScaler(with_mean=False)\nmodelize_LogReg([LogisticRegression],norm.fit_transform(X_t3gram_vectorizer_train),y_cora_train,norm.transform(X_t3gram_vectorizer_test),y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize_LogReg(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(C= 3.3855421686746987, penalty= 'l2').fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(C=3.3855421686746987, penalty='l2'))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=42)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),\n                balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\nnorm=StandardScaler(with_mean=False)\nmodelize_LogReg([LogisticRegression],norm.fit_transform(X_t3gram_vectorizer_train),y_cora_train,norm.transform(X_t3gram_vectorizer_test),y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Nouvel Echantillonage  \ncora_data_neg_sample = cora_data[cora_data['target'] == 1] #Negatives comments\ncora_data_positive_sample = cora_data[cora_data['target'] == 0].reindex()  #Positive Comments\n\ncora_resampling = pd.concat([pd.DataFrame(cora_data_positive_sample.sample(20000)),\n                               pd.DataFrame(cora_data_neg_sample)])\n100*(cora_resampling.groupby('target')['question_text'].count())/cora_resampling['target'].count()\n\n#Cleaning the data \ncora_resampling['clean_question'] = cora_resampling['question_text'].apply(clean_str)\n#Tokenizing and stopwords removing\ncora_resampling['tokeniZ_stopWords_question'] = cora_resampling['clean_question'].apply(tokeniZ_stopWords)\n#Words Stemming\ncora_resampling['stemming_question'] = [[ps.stem(word) for word in words] for words in cora_resampling['tokeniZ_stopWords_question'] ]\ncora_resampling['stemming_question_for_tfidf'] = [' '.join(words) for words in cora_resampling['stemming_question']] \n\nX_cora_train, X_cora_test, y_cora_train, y_cora_test = train_test_split(\n            cora_resampling['stemming_question_for_tfidf']\n            ,cora_resampling['target'], \n            test_size=0.3, random_state=42)\nX_cora_train.shape, X_cora_test.shape, y_cora_train.shape, y_cora_test.shape\n\n#Vectorization with tf-idf\nX_Singt3gram_vectorizer_train = st3gram_vectorizer.fit_transform(X_cora_train)\nX_Singt3gram_vectorizer_test  = st3gram_vectorizer.transform(X_cora_test)\n\nX_t3gram_vectorizer_train = t3gram_vectorizer.fit_transform(X_cora_train)\nX_t3gram_vectorizer_test  = t3gram_vectorizer.transform(X_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#OPTIMISATION DES HYPERPARAMETRES DU MODEL LINEARSVC AVEC GRIDSEARCH - We want to increase the regularization of the classifier (decrease parameter C of LinearSVC)\nfrom sklearn.model_selection import GridSearchCV\nparam_grid={'C': np.linspace(1, 10, 50), 'penalty': ['l2']}\nmodel = LogisticRegression(n_jobs=-1)\nkf = KFold(n_splits=3,random_state=random_state)\n\ngs=GridSearchCV(model, param_grid, scoring='accuracy', n_jobs=-1, cv=kf, verbose=False)\ngs.fit(X_t3gram_vectorizer_train,y_cora_train)\ngs_test_score=balanced_accuracy_score(y_cora_test, gs.predict(X_t3gram_vectorizer_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Accuracy on training sample: {:.3f} with hyperparameters {}\".format(gs.best_score_, gs.best_params_))\n\ngs_test_scorer=accuracy_score(y_cora_test, gs.predict(X_t3gram_vectorizer_test))\nprint(\"Best Accuracy on validation sample: {:.3f} \".format(gs_test_scorer))\nprint(\"Best balanced Accuracy on validation sample: {:.3f} \".format(gs_test_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.metrics import balanced_accuracy_score\n\n#Let's try to increase the data regularization by using the Standardization method\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=30)\ndef modelize_LogReg(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(C=3.272727272727273).fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(C=3.272727272727273))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Balanced Accuracy Score    F1 Score                Balanced Accuracy Score    F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='balanced_accuracy')\n        print('=============================================')\n    \nmodelize_LogReg([LogisticRegression],X_t3gram_vectorizer_train,y_cora_train,X_t3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nnorm=StandardScaler(with_mean=False)\nmodelize_LogReg([LogisticRegression],norm.fit_transform(X_t3gram_vectorizer_train),y_cora_train,norm.transform(X_t3gram_vectorizer_test),y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nselector = SelectPercentile(f_classif,percentile=30)\nmodelize_LogReg([LogisticRegression],normalize(X_t3gram_vectorizer_train),y_cora_train,normalize(X_t3gram_vectorizer_test),y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrandom_state=42\nkf = KFold(n_splits=3,random_state=random_state)\n\nsearchCV = LogisticRegressionCV(\n        Cs=list(np.linspace(0.0001, 100, 250))\n        ,penalty='l1'\n        ,scoring='accuracy'\n        ,cv=kf\n        ,random_state=random_state\n        ,max_iter=10000\n        ,fit_intercept=True\n        ,solver='saga'\n        ,tol=10\n    )\nsearchCV.fit(X_Singt3gram_vectorizer_train, y_cora_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Max accuracy training sample:', searchCV.scores_[1].mean(axis=0).max())\nprint('Max accuracy validation sample:', accuracy_score(searchCV.predict(X_Singt3gram_vectorizer_test),y_cora_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.metrics import balanced_accuracy_score\n\n#Let's try to increase the data regularization by using the Standardization method\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize_LogReg(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf.fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf)]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=51)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Balanced Accuracy Score    F1 Score                Balanced Accuracy Score    F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='balanced_accuracy')\n        print('=============================================')\n    \nmodelize_LogReg([LogisticRegressionCV(Cs=list(np.linspace(0.0001, 100, 250)),penalty='l2',scoring='accuracy',cv=kf\n     ,random_state=random_state,max_iter=10000,fit_intercept=True,solver='newton-cg',tol=10)],X_Singt3gram_vectorizer_train,y_cora_train,X_Singt3gram_vectorizer_test,y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Lets generate the learning curve of the optimized model\nselector = SelectPercentile(f_classif,percentile=70)\ndef modelize_LogReg(list_clf,X,y,X_test,y_test):\n    for clf in list_clf:\n        Clf1 = clf(C= 3.3855421686746987, penalty= 'l2').fit(X,y)\n        Clf2 = Pipeline([('Feature Selection',selector),('Classification',clf(C=3.3855421686746987, penalty='l2'))]).fit(X,y)\n        cv = StratifiedShuffleSplit(n_splits=3 , test_size=.3, random_state=42)\n        print('Model : %s' %type(Clf1).__name__)\n        print('With all features                                              /    With 70% of the best features')\n        print('                 Accuracy Score             F1 Score                Accuracy Score             F1 Score')\n        print('training :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y,Clf1.predict(X)),f1_score(y,Clf1.predict(X),average='weighted'),\n                balanced_accuracy_score(y,Clf2.predict(X)),f1_score(y,Clf2.predict(X),average='weighted')))\n        print('Test     :       %f                   %f           /    %f                   %f' \n              %(balanced_accuracy_score(y_test,Clf1.predict(X_test)),f1_score(y_test,Clf1.predict(X_test),average='weighted')\n                ,balanced_accuracy_score(y_test,Clf2.predict(X_test)),f1_score(y_test,Clf2.predict(X_test),average='weighted')))\n        plot_learning_curve(Clf1, X, y, Clf2, cv, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring='accuracy')\n        print('=============================================')\nnorm=StandardScaler(with_mean=False)\nmodelize_LogReg([LogisticRegression],norm.fit_transform(X_t3gram_vectorizer_train),y_cora_train,norm.transform(X_t3gram_vectorizer_test),y_cora_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrandom_state=42\nkf = KFold(n_splits=3,random_state=random_state)\n\nsearchCV = LogisticRegressionCV(\n        Cs=list(np.linspace(0.0001, 100, 250))\n        ,penalty='l1'\n        ,scoring='accuracy'\n        ,cv=kf\n        ,random_state=random_state\n        ,max_iter=10000\n        ,fit_intercept=True\n        ,solver='saga'\n        ,tol=10\n    )\nsearchCV.fit(X_Singt3gram_vectorizer_train, y_cora_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Max accuracy training sample:', searchCV.scores_[1].mean(axis=0).max())\nprint('Max accuracy validation sample:', accuracy_score(searchCV.predict(X_Singt3gram_vectorizer_test),y_cora_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrandom_state=42\nkf = KFold(n_splits=3,random_state=random_state)\n\nsearchCV = LogisticRegressionCV(\n        Cs=list(np.linspace(0.0001, 100, 250))\n        ,penalty='l2'\n        ,scoring='accuracy'\n        ,cv=kf\n        ,random_state=random_state\n        ,max_iter=10000\n        ,fit_intercept=True\n        #,solver='saga'\n        ,tol=10\n    )\nsearchCV.fit(norm.fit_transform(X_Singt3gram_vectorizer_train), y_cora_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Max accuracy training sample:', searchCV.scores_[1].mean(axis=0).max())\nprint ('Max accuracy validation sample:', accuracy_score(searchCV.predict(norm.transform(X_Singt3gram_vectorizer_test)),y_cora_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"searchCV.get_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Conclusion"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import cross_val_score, StratifiedShuffleSplit, KFold\nfrom sklearn.feature_selection import SelectPercentile\nfrom hyperopt import hp,fmin,Trials, tpe, STATUS_FAIL, STATUS_OK, space_eval, anneal\nfrom hyperopt.pyll import scope\nfrom hyperopt.pyll import stochastic\n\nfrom sklearn.metrics import accuracy_score, f1_score, balanced_accuracy_score\nfrom sklearn.decomposition import SparsePCA\nfrom sklearn.preprocessing import normalize, StandardScaler\n\nimport time\nrandom_state = 42\nkf = KFold(n_splits=2,random_state=random_state)\nn_iter= 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport time\nrandom_state = 42\n\nkf = KFold(n_splits=2,random_state=random_state)\nn_iter= 50\nModel_final_MLPClassifier = Pipeline([('Feature Selection',SelectPercentile(f_classif,percentile=70)),('Classification',MLPClassifier(random_state=random_state, activation = str(space_eval(space,best)['activation'] ),\n                                                 solver = str(space_eval(space,best)['solver']),\n                                                 alpha = int(best['alpha']),\n                                                 hidden_layer_sizes=tuple(space_eval(space,best)['hidden_layer_sizes']),\n                                                 learning_rate = str(space_eval(space,best)['learning_rate'] )))]).fit(X_t3gram_vectorizer_train_,y_cora_train_)\n\nModel_final_LogReg = Pipeline([('Feature Selection',SelectPercentile(f_classif,percentile=70)),('Classification',LogisticRegression(C=3.272727272727273))]).fit(X_t3gram_vectorizer_train_,y_cora_train_)\nModel_final_LinearSVC = Pipeline([('Feature Selection',SelectPercentile(f_classif,percentile=70)),('Classification',LinearSVC(C=0.4056225493975904))]).fit(X_Singt3gram_vectorizer_train_,y_cora_train_)\nModel_final_ExtraTreesClassifier = ExtraTreesClassifier(max_depth= 443.0, max_features= 'sqrt', min_samples_split= 5, n_estimators=117).fit(X_t3gram_vectorizer_train_,y_cora_train_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#mATRICES DE CONFUSIONS\nfrom sklearn.metrics import confusion_matrix\ncm_ETree = confusion_matrix(y_cora_test_, Model_final_ExtraTreesClassifier.predict(X_t3gram_vectorizer_test_))\ncm_LinearSVC = confusion_matrix(y_cora_test_, Model_final_LinearSVC.predict(X_Singt3gram_vectorizer_test_))\ncm_LogReg = confusion_matrix(y_cora_test_, Model_final_LogReg.predict(X_t3gram_vectorizer_test_))\ncm_MLP = confusion_matrix(y_cora_test_, Model_final_MLPClassifier.predict(X_t3gram_vectorizer_test_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#MATRICES DE CONFUSIONS NORMALISEES\ncm_ExtraTreesClassifier = cm_ETree.astype('float') / cm_ETree.sum(axis=1)[:, np.newaxis]\ncm_LinearSVC = cm_LinearSVC.astype('float') / cm_LinearSVC.sum(axis=1)[:, np.newaxis]\ncm_LogisticRegression = cm_LogReg.astype('float') / cm_LogReg.sum(axis=1)[:, np.newaxis]\ncm_MLPClassifier = cm_MLP.astype('float') / cm_MLP.sum(axis=1)[:, np.newaxis]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils.multiclass import unique_labels\nclasses = unique_labels(y_cora_test_)\n#dict_models = {'cm_ExtraTreesClassifier' : 'ExtraTreesClassifier', 'cm_LinearSVC': 'LinearSVC', 'cm_LogisticRegression': 'LogisticRegression', 'cm_MLPClassifier': 'MLPClassifier'}\n\nfig, ((ax1, ax2),(ax3,ax4)) = plt.subplots(2,2, figsize=(10,10))\naX = [ax1, ax2,ax3,ax4]\ndict_models = ['ExtraTreesClassifier',  'LinearSVC',  'LogisticRegression',  'MLPClassifier']\nfor IDX, i in enumerate([cm_ExtraTreesClassifier, cm_LinearSVC, cm_LogisticRegression, cm_MLPClassifier]):\n    im = aX[IDX].imshow(i, interpolation='nearest', cmap=plt.cm.Blues)\n    #aX[IDX].figure.colorbar(im, aX[IDX]=aX[IDX])\n\n    # We want to show all ticks...\n    aX[IDX].set(xticks=np.arange(i.shape[1]), yticks=np.arange(i.shape[0]),\n           xticklabels=classes, yticklabels=classes, # ... and label them with the respective list entries\n           title= dict_models[IDX],\n           ylabel='True label',\n           xlabel='Predicted label')\n    \n    # Rotate the tick labels and set their alignment.\n    #plt.setp(aX[i].get_xticklabels(), rotation=45, ha=\"right\",rotation_mode=\"anchor\")\n    fmt = '.2f' if normalize else 'd'\n    thresh = i.max() / 2.\n    for t in range(i.shape[0]):\n        for j in range(i.shape[1]):\n            aX[IDX].text(j, t, format(i[t, j], fmt),\n                    ha=\"center\", va=\"center\",\n                    color=\"white\" if i[t, j] > thresh else \"black\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm_MLPClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import auc\nfpr1, tpr1, thresholds1 = roc_curve(y_cora_test_, Model_final_MLPClassifier.predict(X_t3gram_vectorizer_test_))\nfpr2, tpr2, thresholds2 = roc_curve(y_cora_test_, Model_final_ExtraTreesClassifier.predict(X_t3gram_vectorizer_test_))\nfpr3, tpr3, thresholds3 = roc_curve(y_cora_test_, Model_final_LinearSVC.predict(X_Singt3gram_vectorizer_test_))\nfpr4, tpr4, thresholds4 = roc_curve(y_cora_test_, Model_final_LogReg.predict(X_t3gram_vectorizer_test_))\n\nprint('AUC - MLPClassifier: %.2f ' %(auc(fpr1, tpr1)))\nprint('AUC - ExtraTreesClassifier: %.2f ' %(auc(fpr2, tpr2)))\nprint('AUC - LinearSVCr: %.2f ' %(auc(fpr3, tpr3)))\nprint('AUC - LogisticRegression: %.2f ' %(auc(fpr4, tpr4)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,10))\nlw = 2\nplt.plot(fpr1, tpr1, #color='darkorange',\n         lw=lw, label='MLPClassifier')\nplt.plot(fpr2, tpr2, #color='bleu', \n         lw=lw, label='ExtratreesClassifier')\nplt.plot(fpr3, tpr3, #color='Olive', \n         lw=lw, label='LinearSVC')\nplt.plot(fpr4, tpr4, #color='red', \n         lw=lw, label='LogisticRegression')\nplt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel(\"Proportion mal classée\")\nplt.ylabel(\"Proportion bien classée\")\nplt.title('ROC')\nplt.legend(loc=\"lower right\")\nplt.savefig('AUC')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  Sommission du modèle final"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(zf_test.open('test.csv'))\nsub_df_target  = pd.read_csv(zf_test.open('sample_submission.csv'))\n#sub_df = pd.read_csv('all/test.csv')\n#sub_df_target = pd.read_csv('all/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df_target.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Cleaning the data \nsub_df['clean_question'] = sub_df['question_text'].apply(clean_str)\n\n#Tokenizing and stopwords removing\nsub_df['tokeniZ_stopWords_question'] = sub_df['clean_question'].apply(tokeniZ_stopWords)\n\n#Words Stemming\nsub_df['stemming_question'] = [[ps.stem(word) for word in words] for words in sub_df['tokeniZ_stopWords_question'] ]\nsub_df['stemming_question_for_tfidf'] = [' '.join(words) for words in sub_df['stemming_question']] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#T3gram questions vectorization\nX_t3gram_vect_sub_test  = t3gram_vectorizer.transform(sub_df['stemming_question_for_tfidf'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#X_t3gram_vect_sub_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Model_final_MLPClassifier = Pipeline([('Feature Selection',SelectPercentile(f_classif,percentile=70)),('Classification',MLPClassifier(random_state=random_state, \n                                                activation = str(space_eval(space,best)['activation'] ),\n                                                 solver = str(space_eval(space,best)['solver']),\n                                                 alpha = int(best['alpha']),\n                                                 hidden_layer_sizes=tuple(space_eval(space,best)['hidden_layer_sizes']),\n                                                 learning_rate = str(space_eval(space,best)['learning_rate'] )))]).fit(X_t3gram_vectorizer_train,y_cora_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['target'] = Model_final_MLPClassifier.predict(X_t3gram_vect_sub_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_submission1 = pd.merge(sub_df,sub_df_target,how='inner',on='qid' )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_submission = (final_submission1[['qid','target']]).rename(columns={'target':'prediction'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_submission.to_csv('final_submission.csv', index=False, sep=',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}