{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport spacy\n\nfrom nltk.corpus import stopwords\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.naive_bayes import MultinomialNB","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-13T02:25:59.555531Z","iopub.execute_input":"2021-10-13T02:25:59.556103Z","iopub.status.idle":"2021-10-13T02:26:02.21102Z","shell.execute_reply.started":"2021-10-13T02:25:59.555991Z","shell.execute_reply":"2021-10-13T02:26:02.210053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Carregando dados de treino\ndf_train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\n\nprint(df_train['target'].value_counts())","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-13T02:26:22.925471Z","iopub.execute_input":"2021-10-13T02:26:22.925808Z","iopub.status.idle":"2021-10-13T02:26:28.508036Z","shell.execute_reply.started":"2021-10-13T02:26:22.925774Z","shell.execute_reply":"2021-10-13T02:26:28.506893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Balanceando os dados para treino\nX_train_insincere = df_train.loc[df_train['target'] == 1]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Sampleando questões sinceras\nsize = round((len(X_train_insincere)*3),1)\n\nX_train_aux = df_train.loc[df_train['target'] == 0]\nX_train_sincere, X_descartar, Y_train_sincere, Y_descartar = train_test_split(X_train_aux['question_text'],\n                                                                       X_train_aux['target'],\n                                                                       train_size=size,\n                                                                       random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T02:31:06.557736Z","iopub.execute_input":"2021-10-13T02:31:06.558053Z","iopub.status.idle":"2021-10-13T02:31:07.178735Z","shell.execute_reply.started":"2021-10-13T02:31:06.558021Z","shell.execute_reply":"2021-10-13T02:31:07.177653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Juntando as questões insinceras com o sample das sinceras\nX_train_npp = pd.concat([X_train_insincere['question_text'], X_train_sincere])\nY_train = pd.concat([X_train_insincere['target'], Y_train_sincere])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Carregando dados de teste\ndf_teste = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\n\nX_test_npp = df_teste['question_text']","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Pré processamento\ndef preprocess(conteudo):\n    nlp = spacy.load('en_core_web_lg')\n    texto_pp = ''\n    en_stop = set(stopwords.words('english'))\n    conteudo_pp = []\n\n    for item in conteudo:\n        pp = nlp(item)\n        for token in pp:\n            if not (token.is_punct or (str(token).lower() in en_stop) or token.like_num):\n                texto_pp += str(token).lower() + ' '\n        conteudo_pp.append(texto_pp)\n        texto_pp = ''\n\n    return conteudo_pp","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Pré processando treino\nX_train_pp = preprocess(X_train_npp)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Pré processando teste\nX_test_pp = preprocess(X_test_npp)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Vetorizando\nvectorizer = TfidfVectorizer(use_idf=True)\ntfidf_model = vectorizer.fit(X_train_pp)\nX_tfidf_train = tfidf_model.transform(X_train_pp)\nX_tfidf_test = tfidf_model.transform(X_test_pp)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Criando modelo\nclf = MultinomialNB()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Treinando\nclf.fit(X_tfidf_train, Y_train)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predizendo\npredicts = clf.predict(X_tfidf_test)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Convertendo resultados para submissão\npredicts = pd.Series(predicts)\n\nsubm_df = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')\nsubm_df['prediction'] = predicts\nsubm_df.to_csv('submission.csv', index=False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]}]}