{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#%% [markdown]\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom wordcloud import WordCloud as wc\nfrom nltk.corpus import stopwords\nimport matplotlib.pylab as pylab\nimport matplotlib.pyplot as plt\nfrom pandas import get_dummies\nimport matplotlib as mpl\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport matplotlib\nimport warnings\nfrom sklearn.ensemble import RandomForestClassifier\nimport sklearn\nimport string\nimport scipy\nimport numpy\nimport nltk\nimport json\nimport sys\nimport csv\nimport os\nnltk.download('averaged_perceptron_tagger')\nnltk.download(\"stopwords\")\n# # Version of the different libraries\nprint('matplotlib: {}'.format(matplotlib.__version__))\nprint('sklearn: {}'.format(sklearn.__version__))\nprint('scipy: {}'.format(scipy.__version__))\nprint('seaborn: {}'.format(sns.__version__))\nprint('pandas: {}'.format(pd.__version__))\nprint('numpy: {}'.format(np.__version__))\nprint('Python: {}'.format(sys.version))\n\nfrom nltk.tokenize import sent_tokenize,word_tokenize\nfrom nltk.corpus import stopwords\ndata = \"All work and no play makes jack dull boy. All work and no play makes jack a dull boy.\"\n\nprint(word_tokenize(data))\nprint(sent_tokenize(data))\n\nfrom nltk.stem import PorterStemmer\nfrom nltk.tokenize import sent_tokenize,word_tokenize\n\nwords=[\"game\",\"gaming\",\"gamed\",\"games\"]\nps=PorterStemmer()\n\nfor word in words:\n    print(ps.stem(word))\n\nfrom nltk.tokenize import PunktSentenceTokenizer\n\nsentences=nltk.sent_tokenize(data)\nfor set in sentences:\n    print(nltk.pos_tag(nltk.word_tokenize(set)))\n\n# # How to make the use of the sns i am not able to get it in poproperly\nsns.set(style='white',context='notebook',palette=\"deep\")\n\n# # EDA\n# ## I will be going to write the diffrent exploratoion technique which can be used to explore the dataset\ntrain=pd.read_csv('../input/train.csv')\ntest=pd.read_csv('../input/test.csv')\nprint('shape of the train',train.shape)\nprint('shape of the test',test.shape)\n\ntrain.size # finding the size of the training set\ntype(train) # tells us about the object type\ntrain.describe() #describe use us about the data\ntrain.sample(5)\n\n# # Data Cleaning\n# # for finding that there is any kind of the null element is present or not(sum of the null values)\ntrain.isnull().sum()\n# # but if we have the null values used it for finding the result in the dataset\nprint('Before Dropping the items',train.shape)\ntrain=train.dropna()\nprint('After droping',train.shape)\n# # for finding the unique items for the target with command below:\n# # getting all the unique from the dataset\ntrain_target=train['target'].values\nnp.unique(train_target)\n\ntrain.head(5)\n\ntrain.tail(5)\n\ntrain.describe()\n\n# Data preprocessing refers to the transformations applied to our data before feeding it to the algorithm.\n\n# Data Preprocessing is a technique that is used to convert the raw data into a clean data set. In other words, whenever the data is gathered from different sources it is collected in raw format which is not feasible for the analysis. there are plenty of steps for data preprocessing and we just listed some of them in general(Not just for Quora) :\n\n#     removing Target column (id)\n#     Sampling (without replacement)\n#     Making part of iris unbalanced and balancing (with undersampling and SMOTE)\n#     Introducing missing values and treating them (replacing by average values)\n#     Noise filtering\n#     Data discretization\n#     Normalization and standardization\n#     PCA analysis\n#     Feature selection (filter, embedded, wrapper)\n#     Etc.\n\n# now we will be going to perfrom some queries on the dataset\n\ntrain.where(train['target']==1).count()\n\n\ntrain[train['target']>1]\n\ntrain.where(train['target']==1).head(5)\n# Imbalanced dataset is relevant primarily in the context of supervised machine learning involving two or more classes.\n# Imbalance means that the number of data points available for different the classes is different: If there are two classes, then balanced data would mean 50% points for each of the class. For most machine learning techniques, little imbalance is not a problem. So, if there are 60% points for one class and 40% for the other class, it should not cause any significant performance degradation. Only when the class imbalance is high, e.g. 90% points for one class and 10% for the other, standard optimization criteria or performance measures may not be as effective and would need modification.\n# Now  we will be going to  explore the exploreing  question\nquestion=train['question_text']\ni=0\nfor q in question[:5]:\n        i=i+1\n        print(\"Question came from the Quora Data_set==\"+q)\n\ntrain[\"num_words\"] = train[\"question_text\"].apply(lambda x: len(str(x).split()))\n# # Some Feature Engineering \nprint(train.columns)\ntrain.head()\n\n# # Count Plot\nax=sns.countplot(x='target',hue='target',data=train,linewidth=5,edgecolor=sns.color_palette(\"dark\",3))\nplt.title('Is data set imbalance')\nplt.show()\n\nax=train['target'].value_counts().plot.pie(explode=[0,0.1],autopct='%1.1f%%',shadow=True)\nax.set_title('target')\nax.set_ylabel('')\nplt.show()\n\ntrain.hist(figsize=(15,20))\nplt.figure()\n\n# #  Creating the histogram which can be used to make the \n# # Making the violin plot\n\nsns.violinplot(data=train,x='target',y='num_words')\n\n\n# # Making the kde plot\nsns.FacetGrid(train,hue=\"target\",size=5).map(sns.kdeplot,\"num_words\").add_legend()\nplt.show()\n\n\n# # Box Plot\ntrain['num_words'].loc[train['num_words']>60]=60\naxes=sns.boxplot(x='target',y='num_words',data=train)\naxes.set_xlabel('Target',fontsize=12)\naxes.set_title(\"No of words in each class\",fontsize=15)\nplt.show()\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}