{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport numpy as np\nimport pandas as pd\nimport spacy","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:01:59.856743Z","iopub.execute_input":"2021-10-15T22:01:59.857765Z","iopub.status.idle":"2021-10-15T22:02:08.293747Z","shell.execute_reply.started":"2021-10-15T22:01:59.857631Z","shell.execute_reply":"2021-10-15T22:02:08.292922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.preprocessing.text import Tokenizer","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:02:08.295172Z","iopub.execute_input":"2021-10-15T22:02:08.295381Z","iopub.status.idle":"2021-10-15T22:02:09.223383Z","shell.execute_reply.started":"2021-10-15T22:02:08.295356Z","shell.execute_reply":"2021-10-15T22:02:09.222629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')[0:10000]\ntest_df = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')[0:10000]","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:02:09.224744Z","iopub.execute_input":"2021-10-15T22:02:09.225380Z","iopub.status.idle":"2021-10-15T22:02:15.128697Z","shell.execute_reply.started":"2021-10-15T22:02:09.225342Z","shell.execute_reply":"2021-10-15T22:02:15.127863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['target'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:02:15.130866Z","iopub.execute_input":"2021-10-15T22:02:15.131141Z","iopub.status.idle":"2021-10-15T22:02:15.145161Z","shell.execute_reply.started":"2021-10-15T22:02:15.131111Z","shell.execute_reply":"2021-10-15T22:02:15.144439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_feature = 20000\n\nX_train = train_df['question_text']\nX_test = test_df['question_text']\n\ny_train = train_df['target']\n","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:02:15.146331Z","iopub.execute_input":"2021-10-15T22:02:15.147083Z","iopub.status.idle":"2021-10-15T22:02:15.152674Z","shell.execute_reply.started":"2021-10-15T22:02:15.147053Z","shell.execute_reply":"2021-10-15T22:02:15.152069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nlp = spacy.load(\"en_core_web_sm\")\n\ndef preprocess(text):\n    doc = nlp(text)\n    tokens = [word.text for word in doc\n            if not word.is_stop\n                and not word.is_space\n                and not word.is_digit\n                and not word.is_punct]\n    return ' '.join(tokens)\n\nX_train = X_train.apply(lambda x: preprocess(x))\nX_test = X_test.apply(lambda x: preprocess(x))","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:02:15.153733Z","iopub.execute_input":"2021-10-15T22:02:15.154482Z","iopub.status.idle":"2021-10-15T22:04:19.944193Z","shell.execute_reply.started":"2021-10-15T22:02:15.154448Z","shell.execute_reply":"2021-10-15T22:04:19.942952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\ntfidf_vectorizer = TfidfVectorizer(use_idf=True)\n\ntrain_vectors = tfidf_vectorizer.fit_transform(X_train).toarray()\ntest_vectors = tfidf_vectorizer.transform(X_test).toarray()","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:04:19.945759Z","iopub.execute_input":"2021-10-15T22:04:19.946204Z","iopub.status.idle":"2021-10-15T22:04:20.662721Z","shell.execute_reply.started":"2021-10-15T22:04:19.946151Z","shell.execute_reply":"2021-10-15T22:04:20.661853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_vectors.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:04:20.663813Z","iopub.execute_input":"2021-10-15T22:04:20.664030Z","iopub.status.idle":"2021-10-15T22:04:20.671379Z","shell.execute_reply.started":"2021-10-15T22:04:20.664004Z","shell.execute_reply":"2021-10-15T22:04:20.670474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import models, layers\n\nmodel = models.Sequential()\nmodel.add(layers.Dense(64, activation='relu', input_shape=(14900,)))\nmodel.add(layers.Dense(64, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n                loss='binary_crossentropy',\n                metrics=['accuracy'])\n\nhistory = model.fit(train_vectors,\n                    y_train,\n                    epochs=20,\n                    batch_size=128,\n                    )","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:04:20.673009Z","iopub.execute_input":"2021-10-15T22:04:20.673358Z","iopub.status.idle":"2021-10-15T22:04:37.769333Z","shell.execute_reply.started":"2021-10-15T22:04:20.673302Z","shell.execute_reply":"2021-10-15T22:04:37.768525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(test_vectors)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:04:37.771997Z","iopub.execute_input":"2021-10-15T22:04:37.772712Z","iopub.status.idle":"2021-10-15T22:04:39.565032Z","shell.execute_reply.started":"2021-10-15T22:04:37.772670Z","shell.execute_reply":"2021-10-15T22:04:39.564124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')[0:10000]\nsubmission['prediction'] = y_pred\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T22:04:39.568644Z","iopub.execute_input":"2021-10-15T22:04:39.568868Z","iopub.status.idle":"2021-10-15T22:04:40.001082Z","shell.execute_reply.started":"2021-10-15T22:04:39.568842Z","shell.execute_reply":"2021-10-15T22:04:40.000230Z"},"trusted":true},"execution_count":null,"outputs":[]}]}