{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"**Read dataset**"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport re\nimport io\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom nltk.stem.porter import PorterStemmer\nfrom fastcache import clru_cache as lru_cache\n\nprint( 'read data' )\ntrain_df = pd.read_csv( '../input/train.csv' )\ntrain_size = len( train_df.index )\ntest_df = pd.read_csv( '../input/test.csv' )\nids_test = test_df[ 'qid' ].values\nclazz_train = train_df[ 'target' ].values\n\nprint( 'training set' )\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62f0149a01620d174c21f4b0acbef73fc23ecdb5"},"cell_type":"code","source":"print( 'test set' )\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e91ce445258397d8a38a87f8b7d363ecbeba1cdf"},"cell_type":"markdown","source":"*Concat all datas*"},{"metadata":{"trusted":true,"_uuid":"e18b36a1d272eefbc81f8ae0f5f5cc902b634a22"},"cell_type":"code","source":"df = pd.concat( [ train_df, test_df ], axis=0, sort=False, ignore_index=True )\n\nprint( 'training set size:', train_size )\nprint( 'test set size:', len(ids_test) )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"295cd2aeb1bb698f62440e22c888ad25be90748f"},"cell_type":"markdown","source":"**Vectorize Question Text**"},{"metadata":{"trusted":true,"_uuid":"e9db836793ef0f4bff00b17ee154fbabd1e5b785"},"cell_type":"code","source":"print( 'vectorize' )\n# stemmer\nstemmer = PorterStemmer()\n@lru_cache(2048)\ndef stem(s):\n\treturn stemmer.stem(s)\nwhitespace = re.compile(r'\\s+')\nnon_letter = re.compile(r'\\W+')\ndef tokenize(text):\n\ttext = text.lower()\n\ttext = non_letter.sub(' ', text)\n\ttokens = []\n\tfor t in text.split():\n\t\tt = stem(t)\n\t\ttokens.append(t)\n\treturn tokens\n\nvectorizer = TfidfVectorizer( use_idf=True, lowercase=True, tokenizer=tokenize )\nvecs = vectorizer.fit_transform( df[ 'question_text' ].values )\nvecs_train = vecs[ :train_size ]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ee92819a0fa3c8c429e0da54c70c98ae0f004fd"},"cell_type":"markdown","source":"*get vocabulary*"},{"metadata":{"trusted":true,"_uuid":"3155417c8311587c25c3fde01304219285346e72"},"cell_type":"code","source":"vocabulary = {v:k for k,v in vectorizer.vocabulary_.items()}\nfor i,k in zip(range(20),vocabulary.keys()):\n    print( k,vocabulary[k] )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b9d1342668781421d22559fcdbda995f105e298"},"cell_type":"markdown","source":"*make sum of positive/negative TFIDVector*"},{"metadata":{"trusted":true,"_uuid":"b487315514d731f3c3879afa1e156e29e2567f2f"},"cell_type":"code","source":"# make weights for class\ncnum = np.sum(clazz_train == 1)\nweight = cnum / ( len(clazz_train) - cnum )\n# sum of each class\nindex = np.arange( train_size )\nindex_posi = index[ clazz_train == 0 ]\nindex_nega = index[ clazz_train == 1 ]\nvecs_posi = vecs_train[ index_posi ].sum( axis=0 ) * ( weight )\nvecs_nega = vecs_train[ index_nega ].sum( axis=0 ) * ( 1.0 - weight)\nprint(vecs_nega[:10])\nprint(vecs_posi[:10])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9df6e00b6c47c818fd431d812584fde35d1b6d69"},"cell_type":"markdown","source":"*make negative score for each word*"},{"metadata":{"trusted":true,"_uuid":"7200079a46a12b546816ad7f54658ff0dd6d1b1e"},"cell_type":"code","source":"# class nega is class value 1\nvecs_score = np.array(vecs_nega - vecs_posi).reshape((-1,))\nprint(vecs_score[:10])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8937d14a527a60ac0b244cd57fc22373a433dd6b"},"cell_type":"markdown","source":"**Show positive/negative words in trainig set**"},{"metadata":{"trusted":true,"_uuid":"7fc1db661163d6bbe7c77f3d59b2d1975512c629"},"cell_type":"code","source":"print( 'positive/negative words in trainig set:' )\nrank = np.argsort( vecs_score )\n# 'positive word' is class 0\nprint( '  positive 20:' )\nfor r in rank[:20]:\n\tprint( vecs_score[ r ], vocabulary[ r ] )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5f9c13afa96bc829af308176b16986c3b3ab2ee"},"cell_type":"code","source":"print( '  negative 20:' )\nfor r in rank[-20:]:\n\tprint( vecs_score[ r ], vocabulary[ r ] )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"12b602319b4350e1c7da9ea99e5a2f4cc0748c24"},"cell_type":"markdown","source":"**Find words that is only in test set**"},{"metadata":{"trusted":true,"_uuid":"8ad7c071f1a33cf6bf43093414359eb8c6dc7d37"},"cell_type":"code","source":"print( 'find words only in test set' )\nvecs_test = vecs[ train_size: ]\ntrain_words = np.nonzero( vecs_train.sum( axis=0 ) )[1]\ntest_words = np.nonzero( vecs_test.sum( axis=0 ) )[1]\nonlytest_words = [ t for t in test_words if t not in train_words ]\nprint( '  words only in test:' )\nfor o in onlytest_words[:20]:\n\tprint( o, vocabulary[ o ] )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b9fd328640771facbde439441bbe8addc0d5c452"},"cell_type":"markdown","source":"**Use nearest word for only in test words**"},{"metadata":{"trusted":true,"_uuid":"0da2238f82ffe73e5318b0c4733979c6a0790ece"},"cell_type":"code","source":"print( 'find nearest word' )\n# load word vector\ndef load_vectors(fname):\n\tfin = io.open(fname, 'r', encoding='utf-8', newline='\\n', errors='ignore')\n\tn, d = map(int, fin.readline().split())\n\tdata = {}\n\tfor line in fin:\n\t\ttokens = line.rstrip().split(' ')\n\t\tif tokens[0] in vectorizer.vocabulary_:\n\t\t\tdata[tokens[0]] = np.array( list( map(float, tokens[1:]) ) )\n\treturn data\n\nword_vec = load_vectors( '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec' )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f8d15fa8b6a5d734cbd28d748e216c1e66ce5f2"},"cell_type":"markdown","source":"*find nearest word in word vector*"},{"metadata":{"trusted":true,"_uuid":"eaf1535229dbf3b4da029d83613dbc59679703cc"},"cell_type":"code","source":"# find nearest word\ndef get_onlytest_words( o ):\n\tresult = 0.0\n\tif vocabulary[ o ] in word_vec:\n\t\tnmin = np.inf\n\t\tnidx = None\n\t\t# use 2000 negative words\n\t\tfor k in rank[-2000:]:\n\t\t\tif k not in onlytest_words and vocabulary[ k ] in word_vec:\n\t\t\t\tn = np.linalg.norm( word_vec[ vocabulary[ o ] ] - word_vec[ vocabulary[ k ] ] )\n\t\t\t\tif nmin > n:\n\t\t\t\t\tnmin = n\n\t\t\t\t\tnidx = k\n\t\tif nidx is not None:\n\t\t\t# use nearest word\n\t\t\tresult = vecs_score[ nidx ]\n\treturn o, result\n# use multiprocessing\nfrom multiprocessing import Pool\nwith Pool(4) as p:\n\tfor o, val in p.map(get_onlytest_words, onlytest_words):\n\t\tvecs_score[ o ] = val","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"751b8bcf6e4ce822d36fcd055ea145b32a1c5430"},"cell_type":"markdown","source":"**Make predict for training data set**"},{"metadata":{"trusted":true,"_uuid":"aa5247a74537d14c5196602500c0a361f68967bc"},"cell_type":"code","source":"print('predict for trainig data set')\nvecs_train_posi = vecs_train[ index_posi ]\nscore_posi = vecs_train_posi.multiply( vecs_score ).sum( axis=1 )\nscore_posi = np.array(score_posi).reshape((-1,))\nprint('posi:',score_posi.min(),score_posi.max(),score_posi.mean(),np.median(score_posi))\n\nvecs_train_nega = vecs_train[ index_nega ]\nscore_nega = vecs_train_nega.multiply( vecs_score ).sum( axis=1 )\nscore_nega = np.array(score_nega).reshape((-1,))\nprint('nega:',score_nega.min(),score_nega.max(),score_nega.mean(),np.median(score_nega))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f27e8cd369ae154a9c4bdeb842bf27265f46657"},"cell_type":"markdown","source":"*make score split point*"},{"metadata":{"trusted":true,"_uuid":"a029c6e11a39afaea93fe7df7abe33b7a309057c"},"cell_type":"code","source":"print('make split point:')\nscore_min = min(score_posi.min(),score_nega.max())\nscore_max = max(score_posi.max(),score_nega.max())\nscore_div = score_min\nscore_div_p = score_div\nscore_gain = score_max - score_min\ncount_true = -np.inf\nfor _ in range(10):\n\tc = len( score_posi[ score_posi<=score_div ] ) + len( score_nega[ score_nega>score_div ] )\n\tif c > count_true:\n\t\tcount_true = c\n\t\tscore_div_p = score_div\n\t\tscore_gain /= 2.0\n\t\tscore_div += score_gain\n\telse:\n\t\tscore_gain /= 2.0\n\t\tscore_div -= score_gain\nprint('split point is',score_div_p)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb4b8f6eadbe69f8f2f0b3b06f12ddcd99f0b792"},"cell_type":"markdown","source":"*scoreing for training data set*"},{"metadata":{"trusted":true,"_uuid":"1d4289763f2073ba9f4e01b73db98f33cf8c0d85"},"cell_type":"code","source":"print('scoreing for training data set')\nTP = len( score_posi[ score_posi<=score_div_p ] )\nFP = len( score_nega[ score_nega<=score_div_p ] )\nFN = len( score_posi[ score_posi>score_div_p ] )\nTN = len( score_nega[ score_nega>score_div_p ] )\nPR = TP / (TP+FP)\nRC = TP / (TP+FN)\nprint('F1 score of training data set is',2*RC*PR / (RC+PR))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"754ecc6ae1162e30bbf7ee5abaef09664cd3e0c7"},"cell_type":"markdown","source":"**Make submission**"},{"metadata":{"trusted":true,"_uuid":"5f10474e0700b6da98f91bbf33337e2931ba3c1c"},"cell_type":"code","source":"score = vecs_test.multiply( vecs_score ).sum( axis=1 )\nscore = np.array(score).reshape((-1,))\nprint(score[:20])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72d1cff0300331a4f6c1a2f759e4ace9df8f84dc"},"cell_type":"markdown","source":"*score to class*"},{"metadata":{"trusted":true,"_uuid":"7a6d811a0e22c9cadaf712ba3f140a00ac9b3741"},"cell_type":"code","source":"score[ score <= 0 ] = 0\nscore[ score > 0 ] = 1\nscore = score.astype( np.int )\nprint(score[:20])\nprint('class0 count:',len(score[ score == 0 ]))\nprint('class1 count:',len(score[ score == 1 ]))\ntest_df[ 'prediction' ] = score\ntest_df[ 'prediction' ] = test_df[ 'prediction' ].astype( np.int )\ntest_df[ [ 'qid','prediction'] ].to_csv( 'submission.csv', index=None )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}