{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from subprocess import check_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf12cfb25e75356478bf1b1393a77eadc03c992c"},"cell_type":"code","source":"train_data = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88c28fdec40bd45d8e66740649103d4648800723"},"cell_type":"code","source":"train_1 = train_data[train_data.target ==1]\ntrain_0 = train_data[train_data.target ==0]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eac0952d4b89aa4fb06c922397d6a215721f21bd"},"cell_type":"code","source":"def split_dataframe_to_chunks(df, n):\n    df_len = len(df)\n    n_rows = df_len//n\n    n_rem = df_len%n\n    count_ = 0\n    dfs = []\n    for count in range(n):\n        if count<n-1:\n            start = count_\n            count_ += n_rows\n            #print(\"%s : %s\" % (start, count))\n            dfs.append(df.iloc[start : count_])\n        else:\n            start = count_\n            count_ += n_rows+n_rem\n            dfs.append(df.iloc[start : count_])\n    return dfs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"156238b4c0504ee65f150744edd86c8c6bc173d4"},"cell_type":"code","source":"train_0_splits = split_dataframe_to_chunks(train_0,5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a89a9cb981a4c4443004b8032045ef75cc2cb05a"},"cell_type":"code","source":"train_0_splits[0].target.value_counts()\n80810/245062","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46b3c86ff667db435ae7f45a2830f7560f869a62"},"cell_type":"code","source":"for i in range(len(train_0_splits)):\n    train_0_splits[i] = train_0_splits[i].append(train_1,ignore_index=True)\n    train_0_splits[i] = train_0_splits[i].sample(frac=1).reset_index(drop=True)\n       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d02b36fcfe93f6e6df3bab2e216b72d3b49a0692"},"cell_type":"code","source":"train_data_list = train_0_splits\ndel train_0_splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"549201107561fc91d1a9879f543b7bdcd27c19fa"},"cell_type":"code","source":"#import libraries\nimport os\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.ensemble import AdaBoostClassifier,RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier, BaggingClassifier\nfrom sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn import svm\nfrom sklearn.decomposition import SparsePCA\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing\nfrom sklearn.metrics import confusion_matrix\nfrom nltk.corpus import stopwords\nimport re\nfrom bs4 import BeautifulSoup\nfrom nltk.stem.wordnet import WordNetLemmatizer, wordnet\nfrom nltk.tokenize import TweetTokenizer, sent_tokenize\nfrom sklearn.metrics import matthews_corrcoef, f1_score\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom imblearn.under_sampling import RandomUnderSampler\nimport time\nfrom sklearn.metrics import cohen_kappa_score\nwordnet_lemmatizer = WordNetLemmatizer()\n# stop_words = stopwords.words('english')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6f15834e8c65dbaedd244e092c4baf47706a6d6"},"cell_type":"code","source":"tokenizer_words = TweetTokenizer()\n\n#clean data\ndef Clean_data(raw_text):\n    #remove HTML\n#     data = BeautifulSoup(raw_text, 'lxml').get_text()\n    data = re.sub(r'http\\S+|www\\S+', '', raw_text)\n    sentences = sent_tokenize(data)\n  #  sentences = sentences[2:-1]\n    clean_data = []\n    for sent in sentences:\n        sent = re.sub(\"[^\\w]\", \" \", sent)\n        sent = re.sub(r\"\\d+\", \" \", sent)\n        sent = re.sub(\"_\", \" \", sent)\n        # sent = re.sub(r\"https|http\", \"\", sent)\n        sent = ' '.join([t.lower() for t in sent.split(' ') if t])\n        sent = ' '.join( [w for w in sent.split() if len(w)>2] )\n        # sent = ' '.join([w for w in sent.split(' ') if w not in ext_stop])\n        sent = ' '.join([wordnet_lemmatizer.lemmatize(t, wordnet.VERB) for t in sent.split()])\n#         sent = ' '.join([w for w in sent.split(' ') if w not in stop_words])\n        clean_data.append(sent)\n    clean_data = '. '.join(clean_data)\n    return clean_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6c07d15c1b30f2972c43af1656eddcc3ebfb97c"},"cell_type":"code","source":"X_ = train_data.question_text\nX_ = [Clean_data(d) for d in X_]\ntrain_data['question_text_clean_with_stop'] = X_\n# train_data.to_csv('../input/train_data_new.csv', index = False)\nX_ = train_data.question_text_clean_with_stop\nY = train_data.target\n# Y = [1 if (y==0) else 0 for y in Y]\n# le = preprocessing.LabelEncoder()\n# Y= le.fit_transform(Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2926c13306c62cd82f438594b14cb2b585877c7"},"cell_type":"code","source":"test_data = pd.read_csv('../input/test.csv')\ntest_X = test_data.question_text\ntest_X = [Clean_data(d) for d in test_X]\ntest_data['question_text_clean_with_stop'] = test_X\ntest_X = test_data.question_text_clean_with_stop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca347e93b0d1c85d5b47fc2f96780f21c595d1cf"},"cell_type":"code","source":"# Y = train_data.target\n# rus = RandomUnderSampler()\n# X, Y = rus.fit_sample(np.array(X).reshape(-1, 1),Y)\n# X = pd.Series([x[0] for x in X])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55767933465e5a048a0cdf57f0fd4208c140b523"},"cell_type":"code","source":"import numpy as np\nfrom scipy import sparse\nfrom sklearn.feature_extraction.text import CountVectorizer\nVOCAB_SIZE = 13500\ncounter = CountVectorizer(max_features=VOCAB_SIZE)\nXc = counter.fit_transform(X_)\ntfidf_vect = TfidfTransformer()\nX_ = tfidf_vect.fit_transform(Xc)\n# clf = LogisticRegression(solver='lbfgs',max_iter=500, multi_class='multinomial')\n# sent_lens =np.sum(Xc, axis=1).astype(\"float\")\n# sent_lens[sent_lens == 0] = 1e-14\n# print(sent_lens.shape)\n# sent_ = np.divide(1,sent_lens)\n# Xc = Xc.multiply(sparse.csr_matrix(sent_))\n# del X\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0da34115342d30cbc51294927c7ce08a7a8e2213"},"cell_type":"code","source":"'''\nfreqs = np.sum(Xc, axis=0).astype(\"float\")\nprobs = freqs / np.sum(freqs)\nALPHA = 1e-3\ncoeff = ALPHA /(ALPHA + probs)\nXw = Xc.multiply(sparse.csr_matrix(coeff))\n# Xw = np.multiply(Xc, coeff)\ndel Xc\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cfecf47f9d2997931096f12222d91dfc58383a8"},"cell_type":"code","source":"'''\nimport os\nGLOVE_EMBEDDINGS = '/kaggle/input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\nE = np.zeros((VOCAB_SIZE, 300))\nfglove = open(GLOVE_EMBEDDINGS, \"r\")\nfor line in fglove:\n    cols = line.strip().split(\" \")\n    word = cols[0]\n    try:\n        i = counter.vocabulary_[word]\n        E[i] = np.array([float(x) for x in cols[1:]])\n    except KeyError:\n        pass\nfglove.close()\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"196f138ee3dc8f14da02bac427ef8d27f7335293"},"cell_type":"code","source":"# E.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eca2c4ac4ca65bd1f456eca468508f93e78a5d61"},"cell_type":"code","source":"'''\n# # Xc = Xc.toarray()\n# # compute word probabilities from corpus\nXs =Xw.dot(sparse.csr_matrix(E))\n# # Xs = np.divide(np.dot(Xw, E), sent_lens)\ndel Xw, E\n# from sklearn.decomposition import TruncatedSVD\n\n# svd = TruncatedSVD(n_components=1, n_iter=20, random_state=0)\n# svd.fit(Xs)\n# # svd.fit(X)\n# pc = svd.components_\n# pc_t = pc.T\n# pc = sparse.csr_matrix(pc)\n# pc_t = sparse.csr_matrix(pc_t)\n# print(type(pc))\n# Xr = Xs - Xs.dot(pc.T).dot(pc) \n# # X_pc = (pc_t).dot(pc)\n# # del pc\n# # Xr_ = X.dot(X_pc)\n# # Xr = X - Xr_\n# del  X,X_pc #Xs,\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8591c9f5644f69c4d4210c467293cb62ee6a4221"},"cell_type":"code","source":"'''\npc = sparse.csr_matrix(pc)\nXr = Xs - Xs.dot(pc.T).dot(pc)\ndel Xs, pc\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27946393e2bf6969880a44f7b6b9ecefb6321654"},"cell_type":"code","source":"'''\nskf = StratifiedKFold(n_splits = 10, shuffle = True)\n# X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size=0.2)\npipeline = Pipeline([\n    ('count_vectorizer',   CountVectorizer(ngram_range = (1,3),max_features=13500)),\n    ('tfidf_transformer',  TfidfTransformer()),\n#     ('SVD', TruncatedSVD(n_components=1, n_iter=10)),\n#     ('PCA', SparsePCA(n_components=1,normalize_components=True)),\n#     ('qda', QuadraticDiscriminantAnalysis(store_covariances=True)),\n#    ('classifier',  svm.SVC())]) #solver='lbfgs',class_weight={0:0.6,1:1.5},max_iter=500, multi_class='multinomial'))])\n    ('classifier', LogisticRegression(solver='lbfgs',max_iter=500, multi_class='multinomial'))])\n#     ('classifier', GradientBoostingClassifier(n_estimators=100, learning_rate=0.01, max_depth=10, random_state=0))])\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d102fbe2365c719ca4acdf5c0cd72bf7016a882d"},"cell_type":"code","source":"'''\naccuracy2 = []\nmcc_value2 =[]\nf1_scores = []\ntest_Y = []\npredicted = []\ncm =[]\nmodel = None\ntfidf_model = None\ncv_model = None\nstart_time = time.time()\nfor train_ix, test_ix in skf.split(X,Y):\n\n    X_train, X_test = X[train_ix], X[test_ix]\n    Y_train, Y_test = Y[train_ix], Y[test_ix]\n   # CV.fit_transform(X_train.values,Y_train)\n#     pipeline.fit(X_train,Y_train)\n    pipeline.fit(X_train.values.astype('U'),Y_train)\n#     cv_model_ =  pipeline.steps[0][1]\n#     tfidf_model_ =  pipeline.steps[1][1]\n#     PCA_model_ = pipeline.steps[2][1]\n#     model_ =  pipeline.steps[3][1]\n    prediction = pipeline.predict(X_test.values.astype('U'))\n    ac = np.mean(prediction == Y_test)\n    mcc = matthews_corrcoef(Y_test, prediction)\n    f1 = f1_score(Y_test, prediction)\n#     if not accuracy2:\n#         model = model_\n#         cv_model = cv_model_\n#         tfidf_model = tfidf_model_\n# #         PCA_model = PCA_model_\n#     elif ac>max(accuracy2) and f1>max(f1_scores):\n#         model = model_\n#         tfidf_model = tfidf_model_\n#         cv_model = cv_model_\n#         PCA_model = PCA_model_\n    test_Y.append(Y_test)\n    predicted.append(prediction)\n    accuracy2.append(ac)\n    mcc_value2.append(mcc)\n    f1_scores.append(f1)\n    print(ac,mcc,f1)\n    cm.append(confusion_matrix(Y_test, prediction))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be6b5a69d3b8cc5ef8a23167d566ee576bc8965d"},"cell_type":"code","source":"# pipelines = {}\npredicted = []\nfor i in range(len(train_data_list)):\n    data = train_data_list[i]\n    X = data.question_text\n    X = [Clean_data(d) for d in X]\n    data['question_text_clean_with_stop'] = X\n    # train_data.to_csv('../input/train_data_new.csv', index = False)\n    X = data.question_text_clean_with_stop\n    Y = data.target\n    accuracy2 = []\n    mcc_value2 =[]\n    f1_scores = []\n#     test_Y = []\n#     predicted = []\n    start_time = time.time()\n#     X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size=0.2)\n    # CV.fit_transform(X_train.values,Y_train)\n    #     pipeline.fit(X_train,Y_train)\n    X = counter.transform(X)\n    X = tfidf_vect.transform(X)\n    clf = LogisticRegression(solver='lbfgs',max_iter=450,class_weight={0:0.8,1:1}, multi_class='multinomial')\n    clf.fit(X,Y)\n    #     cv_model_ =  pipeline.steps[0][1]\n    #     tfidf_model_ =  pipeline.steps[1][1]\n    #     PCA_model_ = pipeline.steps[2][1]\n    #     model_ =  pipeline.steps[3][1]\n#     prediction = pipeline.predict(X_test.values.astype('U'))\n#     ac = np.mean(prediction == Y_test)\n#     mcc = matthews_corrcoef(Y_test, prediction)\n#     f1 = f1_score(Y_test, prediction)\n    #     if not accuracy2:\n    #         model = model_\n    #         cv_model = cv_model_\n    #         tfidf_model = tfidf_model_\n    # #         PCA_model = PCA_model_\n    #     elif ac>max(accuracy2) and f1>max(f1_scores):\n    #         model = model_\n    #         tfidf_model = tfidf_model_\n    #         cv_model = cv_model_\n    #         PCA_model = PCA_model_\n#     model =  pipeline.steps[2][1]\n#     tfidf_model =  pipeline.steps[1][1]\n#     cv_model =  pipeline.steps[0][1]\n#     test_Y.append(Y_test)\n#     predicted.append(prediction)\n#     accuracy2.append(ac)\n#     mcc_value2.append(mcc)\n#     f1_scores.append(f1)\n#     print(ac,mcc,f1)\n#     cm.append(confusion_matrix(Y_test, prediction))\n    test_x = counter.transform(test_X)\n    test_x = tfidf_vect.transform(test_x)\n    predicted_= clf.predict(test_x)\n    predicted.append(predicted_)\n#     pipelines[i]=pipeline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd2380d8bd3e28f131148ea119b8349c333c122c"},"cell_type":"code","source":"from collections import Counter\nprediction = []\nfor i in range(len(predicted[0])):\n    prediction.append([predicted[j][i] for j in range(len(predicted))])\n# max_key = max(stats, key=lambda k: stats[k])\nprediction_ = [max(x, key=lambda k: x[k]) for x in prediction]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74c63427fc2ac4ec8fd625e49ae6efd75b88dfe5"},"cell_type":"code","source":"# model =  pipeline.steps[2][1]\n# tfidf_model =  pipeline.steps[1][1]\n# cv_model =  pipeline.steps[0][1]\n# # for i in model.classes_:\n# feat_ind = np.argsort(model.coef_[0])[::-1][:1000]\n# feat_ = [cv_model.get_feature_names()[idx] for idx in feat_ind]\n# important_features[1]= feat_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e95f71713756a95c27f6bc14bca7f54bc1670fb1"},"cell_type":"code","source":"# del model, tfidf_model, cv_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc9a451df8297fbf2bfbf8336a154ff2275f8306"},"cell_type":"code","source":"# import json\n# with open('important_features.json', 'wb') as f:\n#     f.write(important_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08a619eed731a19c25b418abff18bbade5828017"},"cell_type":"code","source":"test_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4953fbb1111cef4222e0a332f4f28675a881717"},"cell_type":"code","source":"import pandas as pd\n# prediction = pipeline.predict(test_X.values.astype('U'))\nsub = pd.DataFrame()\nsub['qid'] = test_data.qid\nsub['prediction'] = prediction_\nsub.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72a860d6a1f52e996101a2bd0a107e7dd49df9db"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}