{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nfrom nltk.corpus import stopwords\nimport string\nfrom nltk.stem.wordnet import WordNetLemmatizer\nfrom nltk.tokenize import word_tokenize, sent_tokenize, TweetTokenizer\nimport re\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn import linear_model\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nwarnings.filterwarnings(\"ignore\")\nstop_words = set(stopwords.words(\"english\"))\npunctuations = string.punctuation\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"491fb5bceb24720bff8719945da5fdebb60e121c"},"cell_type":"code","source":"#loading csv file\ntrain_data=pd.read_csv(\"../input/train.csv\")\ntest_data=pd.read_csv(\"../input/test.csv\")\nprint(\"Train Shape\",train_data.shape)\nprint(\"Test Shape\",test_data.shape)\ntrain_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b69d9a9605b773786a0993f39e1b489a0d5e44a6"},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d998743bccd94e9854fd8b2b71b81a2742b0ac7"},"cell_type":"code","source":"#data information & description for train data\nprint(\"\\n train data info\")\ntrain_data.info()\nprint(\"\\n train data description\")\nprint(train_data.describe())\n#data information & description for test data\nprint(\"\\n test data info\")\ntest_data.info()\nprint(\"\\n test data description\")\nprint(test_data.describe())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e15b47621e38219a3b5cd36764f134198d3308b4"},"cell_type":"code","source":"# 1 - Filtering target values of Pandas train data frame\n#calculating target 0&1 counts for train data\nx = train_data['target'].value_counts()[0]\nprint(\"train data target 0 count is\",x)\ny = train_data['target'].value_counts()[1]\nprint(\"train data target 1 count is\",y)\nprint('{}% of the questions in the train set are tagged as insincere.'.format((y*100/(x + y)).round(2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccde05963341b1732428153ed35d4199de79eb1f"},"cell_type":"code","source":"#plotting fig\nfig, ax = plt.subplots()\ng = sns.countplot(train_data.target)\ng.set_xticklabels(['Sincere(0)', 'Insincere(1)'])\ng.set_yticklabels([''])\nsincere = train_data[train_data[\"target\"] == 0]\ninsincere = train_data[train_data[\"target\"] == 1]\n# function to show sincere and insincere on bars\ndef show_values_on_bars(axs):\n    def _show_on_single_plot(ax):\n        for p in ax.patches:\n            _x = p.get_x() + p.get_width() / 2\n            _y = p.get_y() + p.get_height()\n            value = '{:.0f}'.format(p.get_height())\n            ax.text(_x, _y, value, ha=\"center\")\n\n    if isinstance(axs, np.ndarray):\n        for idx, ax in np.ndenumerate(axs):\n            _show_on_single_plot(ax)\n    else:\n        _show_on_single_plot(axs)\nshow_values_on_bars(ax)\n\nsns.despine(left=True, bottom=True)\nplt.xlabel('')\nplt.ylabel('')\nplt.title('Distribution of Questions', fontsize=30)\nplt.tick_params(axis='x', which='major', labelsize=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5812c6473cb98bf3d455e57a61f1786938a13a5"},"cell_type":"code","source":"#finding Question length for training data\ntrain_data[\"quest_len\"] = train_data[\"question_text\"].apply(lambda x: len(x.split()))\nprint(\"\\nTrain data with new column quest_length\")\nprint(train_data.head())\nmax_ql_train = train_data[\"quest_len\"].max()\nprint(\"maximum question length is\",max_ql_train)\nmin_ql_train=train_data[\"quest_len\"].min()\nprint(\"minimum question length is\",min_ql_train)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bc3109aef0a47aa8f945be6a3a5539cf7448bd2"},"cell_type":"code","source":"#finding Question length for test data\ntest_data[\"quest_len\"] = test_data[\"question_text\"].apply(lambda x: len(x.split()))\nprint(\"\\nTest data with new column quest_length\")\nprint(test_data.head())\nmax_ql_test = test_data[\"quest_len\"].max()\nprint(\"maximum question length is\",max_ql_test)\nmin_ql_test=test_data[\"quest_len\"].min()\nprint(\"minimum question length is\",min_ql_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81fc632efa722d83428759d6de9df0c630d15b19"},"cell_type":"code","source":"#Finding stop_words\nlem = WordNetLemmatizer()\ntokenizer = TweetTokenizer()\n\ndef clean_text(question):\n    question = question.lower()\n    question = re.sub(\"\\\\n\", \"\", question)\n    question = re.sub(\"\\'\", \"\", question)\n    words = tokenizer.tokenize(question)\n    words = [lem.lemmatize(word, \"v\") for word in words]\n    words = [w for w in words if w not in stop_words and w not in punctuations]\n    clean_sent = \" \".join(words)\n    return clean_sent\ntrain_data[\"clean_question_text\"] = train_data[\"question_text\"].apply(lambda question: clean_text(question))\ntest_data[\"clean_question_text\"] = test_data[\"question_text\"].apply(lambda question: clean_text(question))\nprint(train_data.head())\nprint(test_data.head())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3315a06bdb3307ae0bc80715981c4a94787b511"},"cell_type":"code","source":"train_question_list = train_data[\"clean_question_text\"]\ntest_question_list = test_data[\"clean_question_text\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03899eb636f493deeb14199f356180ad6cd84004"},"cell_type":"code","source":"#CountVectorizer--converting text document into spare matrix\nvectorizer  = CountVectorizer()\nx_train =  vectorizer.fit_transform(train_question_list)\nx_test =  vectorizer.transform(test_question_list)\ny_train_tfidf = np.array(train_data[\"target\"].tolist())\ntrain_x, validate_x, train_y, validate_y = train_test_split(x_train, y_train_tfidf, test_size=0.3)\n\n#Applying multinomial LogisticRegression Algorithm\nmul_lr = linear_model.LogisticRegression(multi_class='multinomial', solver='sag').fit(train_x, train_y)\ny_vad = mul_lr.predict(validate_x)\n\n#calculating accuracy for logistic regression algorithm\nprint('accuracy = %.2f%%' % \\\n      (accuracy_score(validate_y, y_vad)*100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b38a61835df524ab31cc18bd80f989da12e48d2e"},"cell_type":"code","source":"#target prediction\ny_predict = mul_lr.predict(x_test)\npredict = pd.DataFrame(data = y_predict, columns=['prediction'])\npredict = predict.astype(int)\nid = test_data['qid']\nid_df = pd.DataFrame(id)\n# Join predicted into result dataframe and write result as a CSV file\nresult = id_df.join(predict)\nresult.to_csv(\"submission.csv\", index = False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20dcc4db75b9a78c1f452a234198b371b267b3a7"},"cell_type":"code","source":"result.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}