{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Refactoring\n\nThis is a refactory and clean up of the [previous kernel](https://www.kaggle.com/marcocarnini/a-minimalistic-approach).\n\n## Read Input"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef get_train(howManyLines):\n    train = pd.read_csv(\"../input/train.csv\", nrows=howManyLines)\n    print(\"Rows read: \", howManyLines)\n    print(\"Insincere ratio: \", np.mean(train.target))\n    return train\n    \ntrain = get_train(10000)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b97e221d0fa826d0bd2203f130db8c080637f500"},"cell_type":"markdown","source":"## Get embedding"},{"metadata":{"trusted":true,"_uuid":"3a6d93bea50ea96b856d287252b1f5b36fe532db"},"cell_type":"code","source":"def get_coefs(word,*arr): \n    return word, np.asarray(arr, dtype='float32')\n\ndef get_embedding(embeddingFile):\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(embeddingFile))\n    print(\"Loaded keys: \", len(set(embeddings_index.keys())))\n    return embeddings_index\n\nembeddings_index = get_embedding('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')\nembedding_size = len(embeddings_index[\"The\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5c57e2e5f169e8efd625cd0387c3b96fdebcc67"},"cell_type":"markdown","source":"## Get vocabulary"},{"metadata":{"trusted":true,"_uuid":"c62338a7710b1a8616c153112d84104f94f18f51"},"cell_type":"code","source":"def get_sentences_list(questions):\n    return [[j for j in i\\\n             .replace(\"?\", \" ? \")\\\n             .replace(\"…\", \" … \")\\\n             .replace(\"/\", \" / \")\\\n             .replace(\":\", \" : \")\\\n             .replace(\",\", \" , \")\\\n             .replace(\"(\", \" ( \")\\\n             .replace(\")\", \" ) \")\\\n             .replace(\"%\", \" % \")\\\n             .replace(\"$\", \" $ \")\\\n             .replace(\"'s\", \" 's \")\\\n             .replace(\"’s\", \" ’s \")\\\n             .replace(\"!!!\", \" !!! \")\\\n             .replace(\"I’m\", \" I'm \")\\\n             .replace('\"The', \" The \")\\\n             .replace('don’t', \" con't \")\\\n             .replace(\"What's\", \" What is \")\\\n             .replace(\" so, \", \" so , \")\\\n             .replace(\" you've \", \" you have \")\\\n             .replace(\" aren't \", \" are not \")\\\n             .replace(\" won't \", \" will not \")\\\n             .replace(\" me, \", \" me , \")\\\n             .replace(\" Isn't \", \" Is not \")\\\n             .replace(\" they're \", \" they are \")\\\n             .replace(\"haven't\", ' have not ')\\\n             .replace(\"can’t\", \" can't \")\\\n             .replace(\"isn't\", \" is not \")\\\n             .replace(\"con't\", \" can't \")\\\n             .replace(\"India,\", \" India , \")\\\n             .replace(\"win,\", \" win , \")\\\n             .replace(\"life,\", \" life , \")\\\n             .replace(\"time,\", \" time , \")\\\n             .replace(\"better,\", \" better , \")\\\n             .replace(\"math]\", \" math ] \")\\\n             .replace(\"people,\", \" people, \")\\\n             .replace('doesn’t', \" doesn't \")\\\n             .replace(\"shouldn't\", \" should not \")\\\n             .replace(\"them,\", \" them , \")\\\n             .replace('\"I', ' \" I ')\\\n             .replace(\"shouldn't\", \" should not \")\\\n             .replace(\"shouldn't\", \" should not \")\\\n             .replace(\"Isn't\", \" Is not \")\\\n             .replace(\"years,\", \" years , \")\\\n             .replace(\"wasn't\", \" was not \")\\\n             .replace(\"couldn't\", \" could not \")\\\n             .replace(\"wouldn't\", \" would not \")\\\n             .replace(\"Quora,\", \" Quora , \")\\\n             .replace(\"I’ve\", \" I've \")\\\n             .replace(\"world,\", \" world , \")\\\n             .replace(\"now,\", \" now , \")\\\n             .replace(\"US,\", \" US , \")\\\n             .replace(\"country,\", \" country , \")\\\n             .replace(\"person,\", \" person , \")\\\n             .replace('you’ve', \" you have \")\\\n             .replace('didn’t', \" didn't \")\\\n             .replace('do,', \" do , \")\\\n             .replace('fight,', \" fight , \")\\\n             .replace('day,', \" day , \")\\\n             .replace('\"the', ' \" the ')\\\n             .replace('school,', ' school , ')\\\n             .replace('China,', ' China , ')\\\n             .replace(\"Shouldn't\", \" Should not \")\\\n             .replace(\"Trump,\", \" Trump , \")\\\n             .replace(\"year,\", \" year , \")\\\n             .replace(\"him,\", \" him , \")\\\n             .replace(\"Also,\", \" Also , \")\\\n             .replace(\"isn’t\", \" is not \")\\\n             .replace(\"work,\", \" work , \")\\\n             .replace(\"today,\", \" today , \")\\\n             .replace(\"that,\", \" that , \")\\\n             .replace(\"Wouldn't\", \" Would not \")\\\n             .replace(\"Aren't\", \" Are not \")\\\n             .replace(\"Doesn't\", \" Does not \")\\\n             .replace(\"money,\", \" money , \")\\\n             .replace(\"ago,\", \" ago , \")\\\n             .replace(\"say,\", \" say , \")\\\n             .replace(\"is,\", \" is , \")\\\n             .replace(\"(if\", \" ( if \")\\\n             .replace(\"up,\", \" up , \")\\\n             .replace(\"we're\", \" we are \")\\\n             .replace(\"Can't\", \" Can not \")\\\n             .replace(\"weren't\", \" were not \")\\\n             .replace('won’t', ' will not ')\\\n             .replace(\"aren’t\", \" are not \")\\\n             .replace(\"2017.\", \" 2017 . \")\\\n             .replace('Quorans', ' Quoran ')\\\n             .replace(\"hasn't\", ' has not ')\\\n             .replace(\"cryptocurrencies\", ' cryptocurrency ')\\\n             .replace(\"Brexit\", ' British exit ')\\\n             .replace(\"Redmi\", \"Xiaomi \")\\\n             .replace(\"they’re\", \" they are \")\\\n             .replace(\"you’re\", \" you're \")\\\n             .replace(\"they've\", \" they have\")\\\n             .replace('\"', ' \" ')\\\n             .replace('C#', ' C++ ')\\\n             .replace('Isn’t', ' Is not ')\\\n             .replace('haven’t', ' have not ')\\\n             .replace(\"'The\", \" ' The \")\\\n             .replace(\"they've\", \" they have\")\\\n             .replace('\"', ' \" ')\\\n             .replace('Isn’t', ' Is not ')\\\n             .replace('haven’t', ' have not ')\\\n             .replace(\"'The\", \" ' The \")\\\n             .replace(\"hadn't\", \" had not \")\\\n             .replace(\"2018.\", \" 2018 . \")\\\n             .replace(\"^2\", \" squared \")\\\n             .replace(\"OnePlus\", \" Chinese Smartphone \")\\\n             .replace('\"what', ' \" what')\\\n             .replace('UCEED', ' CEED ')\\\n             .replace('engineering.', ' engineering .')\\\n             .replace('wouldn’t', ' would not ')\\\n             .replace(\"We're\", \"We are\")\\\n             .replace('it\"', ' it \" ')\\\n             .replace(\"wasn’t\", \"was not\")\\\n             .replace(\"GDPR\", \" General Data Protection Regulation \")\\\n             .replace(\"we've\", \" we have \")\\\n             .replace(\"Blockchain\", \" blockchain \")\\\n             .replace('\"Why', ' \" Why ')\\\n             .replace('Coinbase', ' cryptocurrency ')\\\n             .replace('shouldn’t', ' should not ')\\\n             .replace('Adityanath', ' yogi ')\\\n             .replace('Don’t', \" Don't \")\\\n             .replace('Machedo', \" Strategist Influencer \")\\\n             .replace('BNBR', \" Be Nice, Be Respectful \")\\\n             .replace(\"'I\", \" ' I \")\\\n             .replace(\"'the\", \" ' the \")\\\n             .replace('“The', ' \" The ')\\\n             .replace('Boruto', ' Naruto ')\\\n             .replace('\"You', ' \" You ')\\\n             .replace(\"would've\", \" would have \")\\\n             .replace(\"You're\", \" You are \")\\\n             .replace(\"ethereum\", \" cryptocurrency \")\\\n             .replace(\"ethereum\", \" cryptocurrency \")\\\n             .replace(\"DCEU\", \" DC Extended Universe \")\\\n             .replace('\"in', ' \" in ')\\\n             .replace('“I', ' \" I')\\\n             .replace('\"to\"', ' \" to')\\\n             .replace('IIEST', \"Indian Institute of Engineering Science and Technology\")\\\n             .replace('couldn’t', ' could not')\\\n             .replace(\"they'll\", \" they will \")\\\n             .replace(\"Wasn't\", \" Was not \")\\\n             .replace('\"My', ' \" My')\\\n             .replace('Qoura', ' Quora ')\\\n             .replace('up\"', ' up \" ')\\\n             .replace('10+2', ' 10 + 2 ')\\\n             .replace('\"no', ' \" no ')\\\n             .replace(\"parents'\", \" parents 's \")\\\n             .replace(\"SJWs\", \" SJW \")\\\n             .replace('life\"', ' life \" ')\\\n             .replace('\\\\frac\"', ' fraction ')\\\n             .replace('\"not', ' not ')\\\n             .replace(\"Qur'an\", \"Quran\")\\\n             .replace('\"When', ' \" When ')\\\n             .replace('people\"', ' people \" ')\\\n             .replace('depression.', ' depression . ')\\\n             .replace('..', ' ... ')\\\n             .replace('x+1', ' x + 1 ')\\\n             .replace('we’re', ' we are ')\\\n             .replace('\"good', ' \" good ')\\\n             .replace('Trump.', ' Trump . ')\\\n             .replace('to\"', ' to \" ')\\\n             .replace(\"he'll\", \" he will \")\\\n             .replace('time\"', ' time \"')\\\n             .replace('\"No', ' \" No')\\\n             .replace('10+', ' 10 +')\\\n             .replace('LNMIIT', ' Institute Of Information Technology ')\\\n             .replace(\"Won't\", ' Will not ')\\\n             .replace(\"Upwork\", ' Freelancing Platform ')\\\n             .replace('out\"', ' out \" ')\\\n             .replace(\"We've\", ' We have ')\\\n             .replace('\"We', ' \" We ')\\\n             .replace('anti-Trump', ' anti Trump ')\\\n             .replace('Kavalireddi', ' Market Expert ')\\\n             .replace('C#', ' dot net ')\\\n             .replace('.net', ' dot net ')\\\n             .replace('.NET', ' dot net ')\\\n             .replace('\"This', ' \" This ')\\\n             .replace('I’ll', \"I'll\")\\\n             .replace('Zerodha', \" discount broker \")\\\n             .replace('university.', \" university . \")\\\n             .replace(\"They're\", \" They are \")\\\n             .replace('world\"', ' world \" ')\\\n             .replace('bhakts', ' bhakt ')\\\n             .replace(\"you'd\", \"you could\")\\\n             .replace(\"Quora.\", \" Quora .\")\\\n             .replace(\"Smartphone\", \" smartphone \")\\\n             .replace('hasn’t', \" has not \")\\\n             .replace('demonetisation', \" demonetization \")\\\n             .replace('Nice,', \" Nice , \")\\\n             .replace('Strategist', \" strategist \")\\\n             .replace(\"Didn't\", \" Did not \")\\\n             .replace(\"alt right\", \" Alt Right \")\\\n             .replace(\"\\\\frac\", \" fraction \")\\\n             .replace(\"Freelancing\", \" freelancing \")\\\n             .replace(\"A+\", \" A + \")\\\n             .replace(\"100+\", \" 100 + \")\\\n             .replace(\"Wouldn’t\", \" Would not \")\\\n             .replace(\"Unacademy\", \" Indian learning platform \")\\\n             .replace(\"anxiety.\", \" anxiety . \")\\\n             .replace(\"Vajiram\", \" Indian institute \")\\\n             .replace(\"Doklam\", \" disputed territory \")\\\n             .replace(\"Delhi.\", \" Dehli . \")\\\n             .replace(\"NICMAR\", \" Indian National Institute of Construction Management and Research \")\\\n             .replace(\"Couldn't\", \" Could not \")\\\n             .replace(\"engineer.\", \" engineer . \")\\\n             .replace(\"I’d\", \" I'd \")\\\n             .replace(\"others'\", \" others 's \")\\\n             .replace(\"countries'\", \" countries 's \")\\\n             .replace('Shouldn’t', 'Should not')\\\n             .replace('chsl', 'Combined Higher Secondary Level')\\\n             .replace('AlShamsi', ' Investor on Quora ')\\\n             .replace(\"they'd\", \"they would\")\\\n             .replace(\"HackerRank\", \"Hacker Rank\")\\\n             .replace(\"visa.\", \" visa . \")\\\n             .replace(\"^3\", \" ^ 3 \")\\\n             .replace(\"weren’t\", \" were not \")\\\n             .replace(\"Doesn’t\", \" Does not \")\\\n             .replace(\"MUOET\", \" Manipal University Online Entrance Test \")\\\n             .replace(\"Help!\", \" Help ! \")\\\n             .replace(\"Awdhesh\", \" Indian writer on Quora \")\\\n             .replace(\"“the\", ' \" the ')\\\n             .replace(\"A2A'd\", ' asked to answer ')\\\n             .replace(\"Litecoin\", ' cryptocurrency ')\\\n             .replace(\"suicide.\", ' suicide . ')\\\n             .replace(\"eLitmus\", ' Indian recruiter company ')\\\n             .replace(\"Jiren\", ' Dragonball character ')\\\n             .replace(\"it'll\", ' it will ')\\\n             .replace(\"Bhakts\", ' bhakt ')\\\n             .replace(\"he'd\", ' he would ')\\\n             .replace(\"abroad.\", ' abroad . ')\\\n             .replace(\"Cryptocurrency\", ' cryptocurrency ')\\\n             .replace(\"b.SC\", ' BSc ')\\\n             .replace(\"could've\", ' could have ')\\\n             .replace(\"#1\", ' # 1 ')\\\n             .replace(\"Ryzen\", ' AMD processors')\\\n             .replace(\"[\\\\\", ' [ \\\\ ')\\\n             .replace(\"+y\", ' + y ')\\\n             .replace(\"=1\", ' = 1 ')\\\n             .replace(\"Jesus'\", \" Jesus ' \")\\\n             .replace(\"[\\\\\", ' [ \\\\ ')\\\n             .replace(\"+y\", ' + y ')\\\n             .replace(\"=1\", ' = 1 ')\\\n             .replace(\"Jesus'\", \" Jesus ' \")\\\n             .replace(\"5'4\", \" 5 inches 4 \")\\\n             .replace(\"you'\", \" you ' \")\\\n             .replace(\"altcoins\", \" cryptocurrency \")\\\n             .replace(\"altcoin \", \" cryptocurrency \")\\\n             .replace(\"Mumbai.\", \" Mumbai . \")\\\n             .replace(\"developer.\", \" developer . \")\\\n             .replace(\"2+\", \" 2 + \")\\\n             .replace(\"disorder.\", \" disorder . \")\\\n             .replace('they’ve', \" they have \")\\\n             .replace('Baahubali', \" Indian epic action film \")\\\n             .replace('-x', \" - x \")\\\n             .replace('1+x', \" 1 + x \")\\\n             .replace(\"US'\", \" US ' \")\\\n             .replace(\"r-aping\", \" raping \")\\\n             .replace(\"SRMJEE\", \" Indian university level entrance test \")\\\n             .replace(\"=0\", \" = 0 \")\\\n             .replace(\"B+\", \" B + \")\\\n             .replace(\"SGSITS\", \" Indian public engineering institution \")\\\n             .replace(\"B+\", \" B + \")\\\n             .replace(\"Thanks!\", \" Thanks + \")\\\n             .replace(\"90+\", \" 90 + \")\\\n             .replace(\"e^\", \" exp ^ \")\\\n             .replace(\"MU-OET\", \" Indian Manipal University Online Entrance Test \")\\\n             .replace(\"Pakistan.\", \" Pakistan . \")\\\n             .replace(\"Beerus\", ' Dragonball character ')\\\n             .replace(\"Skripal\", ' Russian military intelligence ')\\\n             .replace(\"it!\", ' it ! ')\\\n             .replace(\"coinbase\", ' cryptocurrency ')\\\n             .replace(\"me !\", ' me ! ')\\\n             .replace(\"Hons.\", ' honors degree ')\\\n             .replace(\"hons.\", ' honors degree ')\\\n             .replace(\"don't.\", \" don't. \")\\\n             .replace(\"000.\", \" 0 0 0 . \")\\\n             .replace(\"20+\", \" 20 + \")\\\n             .replace(\"{x\", \" { x \")\\\n             .replace(\"5+\", \" 5 + \")\\\n             .replace(\"1+\", \" 1 + \")\\\n             .replace(\"{1}\", \" { 1 } \")\\\n             .replace(\"you’d\", \" you would \")\\\n             .replace(\"we'll\", \" we will \")\\\n             .replace(\"ain't\", \" be not \")\\\n             .replace(\"BMSCE\", \" Indian College of Engineering \")\\\n             .replace(\"Aren’t\", \" Are not \")\\\n             .replace(\"you!\", \" you ! \")\\\n             .replace(\"a+b\", \" a + b \")\\\n             .replace(\"Gurugram\", \" Gurgaon \")\\\n             .replace(\"2016.\", \" 2016 . \")\\\n             .replace(\"+c\", \" + c \")\\\n             .replace(\"Amazon.in\", \" Amazon India \")\\\n             .replace(\"\\\\sqrt\", \" \\\\ sqrt \")\\\n             .replace(\"me!\", \" me ! \")\\\n             .replace(\"don't.\", \" don't . \")\\\n             .replace(\"3+\", \" 3 + \")\\\n             .replace('bahubali', \" Indian epic action film \")\\\n             .replace(\"200+\", \" 200 + \")\\\n             .replace(\"What\\u200b\", \" What \")\\\n             .replace(\"=x\", \" = x \")\\\n             .replace(\"Zebpay\", \" cryptocurrency \")\\\n             .replace(\"litecoin\", \" cryptocurrency \")\\\n             .replace(\"5'10\", \" 5 inches 10 \")\\\n             .replace(\"5'11\", \" 5 inches 11 \")\\\n             .replace(\"5'5\", \" 5 inches 5 \")\\\n             .replace(\"5'6\", \" 5 inches 6 \")\\\n             .replace(\"5'7\", \" 5 inches 7 \")\\\n             .replace(\"5'8\", \" 5 inches 8 \")\\\n             .replace(\"5'9\", \" 5 inches 9 \")\\\n             .replace(\"5'3\", \" 5 inches 3 \")\\\n             .replace(\"5'2\", \" 5 inches 2 \")\\\n             .replace(\"5'\", \" 5 inches \")\\\n             .replace(\"graduate.\", \" graduate . \")\\\n             .replace(\"Alshamsi\", \" investor in Quora \")\\\n             .replace(\"Ravula\", \" trainer on youtube \")\\\n             .replace(\"Binance\", \" cryptocurrency \")\\\n             .replace(\"C#\", \" C++ \")\\\n             .replace(\"Instagram.\", \" Instagram . \")\\\n             .replace(\"Golang\", \" Google programming language \")\\\n             .replace(\"BIPC\", \" university course \")\\\n             .replace(\"x^\", \" x ^ \")\\\n             .replace(\"^1\", \" ^ 1 \")\\\n             .replace(\"^n\", \" ^ n \")\\\n             .replace(\"You've\", \" You have \")\\\n             .replace(\"OBC-NCL\", \" Indian cast religion income \")\\\n             .replace(\"1+1\", \" 1 + 1 \")\\\n             .replace(\"\\\\to\", \" \\\\ to \")\\\n             .replace(\"graduation.\", \" graduation . \")\\\n             .replace(\"}[\", \" } [ \")\\\n             .replace(\".I\", \" . I \")\\\n             .replace(\"\\\\lim_\", \" \\\\ limit \")\\\n             .replace(\"Swachh\", \" Indian campaign \")\\\n             .replace(\"MHCET\", \" Indian University Entrance Test \")\\\n             .replace(\"Bangalore.\", \" Bangalore . \")\\\n             .replace(\"who've\", \" who have \")\\\n             .replace(\"josaa\", \" Indian University Entrance Test \")\\\n             .replace(\"passport.\", \" passport . \")\\\n             .replace(\"'a\", \" ' a \")\\\n             .replace(\"\\\\displaystyle\", \" \\\\ display \")\\\n             .replace(\".How\", \" . How \")\\\n             .replace(\"Truecaller\", \" mobile app \")\\\n             .replace(\"non-\", \" non - \")\\\n             .replace(\"studying.\", \" studying . \")\\\n             .replace(\"startup.\", \" startup . \")\\\n             .replace(\"EVM'S\", \" Indian Electronic Voting Machines 's \")\\\n             .replace(\"divorce.\", \" divorce . \")\\\n             .replace(\"50+\", \" 50 + \")\\\n             .replace(\"40+\", \" 40 + \")\\\n             .replace(\"salary.\", \" salary . \")\\\n             .replace(\"OBOR\", \" One Belt One Road \")\\\n             .replace(\"300+\", \" 300 + \")\\\n             .replace(\"m.SC\", \" MSc \")\\\n             .replace(\"Google+\", \" Google + \")\\\n             .replace(\"\\\\int\", \" \\\\ integral \")\\\n             .replace(\"upwork\", \" Freelancing Platform \")\\\n             .replace(\"\\\\right\", \" \\\\ right \")\\\n             .replace(\"Chromecast\", \" Google digital media player \")\\\n             .replace(\"500+\", \" 500 + \")\\\n             .replace(\"TensorFlow\", \" Google machine learning platform \")\\\n             .replace(\"n+1\", \" n + 1 \")\\\n             .replace(\"#MeToo\", \" Movement against sexual harassment \")\\\n             .replace(\"Indian.\", \" Indian . \")\\\n             .replace(\"60+\", \" 60 + \")\\\n             .replace(\"friends'\", \" friends 's \")\\\n             .replace(\"iisc\", \" Indian public institute \")\\\n             .replace(\"'to\", \" ' to \")\\\n             .replace(\"What're\", \" What are \")\\\n             .replace(\"0+\", \" 0 + \")\\\n             .replace(\"Muslim.\", \" Muslim . \")\\\n             .replace(\".what\", \" . what \")\\\n             .replace(\"adhaar\", \" Indian identity number \")\\\n             .replace(\"Adhaar\", \" Indian identity number \")\\\n             .replace(\"y'all\", \" you all \")\\\n             .replace(\"engg.\", \" engineering \")\\\n             .replace(\"Islam.\", \" Islam . \")\\\n             .replace(\"\\\\left\", \" \\\\ left \")\\\n             .replace(\"+b\", \" + b \")\\\n             .replace(\"Pune.\", \" Pune . \")\\\n             .replace(\"clickbait\", \" attention grabbing \")\\\n             .replace(\"USICT\", \" Indian University \")\\\n             .replace(\"8+\", \" 8 + \")\\\n             .replace(\"Explain.\", \" Explain . \")\\\n             .replace(\"major.\", \" major . \")\\\n             .replace(\"PESSAT\", \" Indian University \")\\\n             .replace(\"demonitisation\", \" demonetization \")\\\n             .replace(\"Muslims.\", \" Muslims . \")\\\n             .replace(\"help!\", \" help ! \")\\\n             .replace(\"\\u200b\", \" \")\\\n             .replace(\"Hyderabad.\", \" Hyderabad . \")\\\n             .replace(\"30+\", \" 30 + \")\\\n             .replace(\"Patreon\", \" membership platform \")\\\n             .replace(\"'a'\", \" ' a ' \")\\\n             .replace(\"maths.\", \" mathematics . \")\\\n             .replace(\"x[\", \" x [ \")\\\n             .replace(\"racist.\", \" racist . \")\\\n             .replace(\"dx[\", \" dx [\")\\\n             .replace(\"Dies™\", \" Dies trademark \")\\\n             .replace(\"microservices\", \" micro services \")\\\n             .replace(\"400+\", \" 400 + \")\\\n             .replace(\"LGBT+\", \" LGBT \")\\\n             .replace(\"unacademy\", \" Indian learning platform \")\\\n             .replace(\"'not\", \" ' not \")\\\n             .replace(\"teenager.\", \" teenager . \")\\\n             .replace(\"pro-Trump\", \" pro Trump \")\\\n             .replace(\"A-\", \" A - \")\\\n             .replace(\"Whst\", \" What \")\\\n             .replace(\"Demonetization\", \" demonetization \")\\\n             .replace(\"y=\", \" y = \")\\\n             .replace(\"{1\", \" 1 { \")\\\n             .replace(\"{2}\", \" { 2 } \")\\\n             .replace(\"{n\", \" { n \")\\\n             .replace(\"c#\", \" c++ \")\\\n             .replace(\"4+\", \" 4 + \")\\\n             .replace(\"O+\", \" O + \")\\\n             .replace(\"O+\", \" O + \")\\\n             .replace(\"\\\\dfrac\", \" \\\\ fraction \")\\\n             .replace(\"would’ve\", \" would have \")\\\n             .replace(\"Koinex\", \" cryptocurrency \")\\\n             .replace(\"Alt-Right\", \" Alt Right \")\\\n             .replace(\"IISERs\", \" Indian Institute of Science Education and Research \")\\\n             .replace(\"}}\", \" } } \")\\\n             .replace(\"fortnite\", \" Fortnite \")\\\n             .replace(\"Williams'\", \" Williams 's \")\\\n             .replace(\"hyperloop\", \" Hyperloop \")\\\n             .replace(\"ReactJS\", \" JavaScript Library \")\\\n             .replace(\"worthless.\", \" worthless . \")\\\n             .replace(\"Didn’t\", \" Did not \")\\\n             .replace(\"Bittrex\", \" blockchain \")\\\n             .replace(\"LGBTQ+\", \" LGBTQ + \")\\\n             .replace(\"Java.\", \" Java . \")\\\n             .replace(\"dating.\", \" dating . \")\\\n             .replace(\"Byju\", \" learning app \")\\\n             .replace(\"“What\", ' \" What ')\\\n             .replace(\"MBA.\", \" MBA . \")\\\n             .replace(\"x+2\", \" x + 2 \")\\\n             .replace(\"LBSNAA\", \" Indian research and training institute \")\\\n             .replace(\"\\\\infty}\", \" \\\\ infinity } \")\\\n             .replace(\"=3\", \" = 3 \")\\\n             .replace(\"2015.\", \" 2015 . \")\\\n             .replace(\"H+\", \" Transhumanist party \")\\\n             .replace(\"Zamasu\", \" Dragonball character \")\\\n             .replace(\"UPSE\", \" UPSC \")\\\n             .replace(\".What\", \" . What \")\\\n             .replace(\"Irodov\", \" Difficult Physics book \")\\\n             .replace(\"tensorflow\", \" Google machine learning platform \")\\\n             .replace(\"FTRE\", \" FIITJEE \")\\\n             .replace(\"Fiitjee\", \" FIITJEE \")\\\n             .replace(\"SSC-CGL\", \" Indian Graduate Level Exam \")\\\n             .replace(\"CSE.\", \" CSE .\")\\\n             .replace(\"uncomfortable.\", \" uncomfortable . \")\\\n             .replace(\"{2\", \" { 2 \")\\\n             .replace(\"'white\", \" ' white \")\\\n             .replace(\"JoSAA\", \"  Indian University Entrance Test  \")\\\n             .replace(\"overweight.\", \" overweight . \")\\\n             .replace(\"[0\", \" [ 0 \")\\\n             .replace(\"Chennai.\", \" Chennai . \")\\\n             .replace(\"₹\", \" Indian Rupee \")\\\n             .replace(\"=2\", \" = 2 \")\\\n             .replace(\"Kalpit\", \" best participant to JEE \")\\\n             .replace(\"Haven't\", \" Have not \")\\\n             .replace(\"crush.\", \" crush . \")\\\n             .replace(\"suicidal.\", \" suicidal . \")\\\n             .replace(\"grades.\", \" grades . \")\\\n             .replace(\"IIITH\", \" Indian Research University \")\\\n             .replace(\"Trump-Russia\", \" Trump - Russia \")\\\n             .replace(\"Codeforces\", \" website for competive programming contests \")\\\n             .replace(\"NLUs\", \" NLU \")\\\n             .replace(\"£1000\", \" £ 1000 \")\\\n             .replace(\"hadn’t\", \" had not \")\\\n             .replace(\"\\\\sin\", \" \\\\ sin \")\\\n             .split(\" \") if j != \"\"] for i in questions]\n\ndef get_vocab(sentences):\n    words = set([item for sublist in sentences for item in sublist])\n    print(\"Number of words:\", len(words))\n    return words\n\nsentences = get_sentences_list(train.question_text)\nmax_words = np.max([len(i) for i in sentences])\nwords = get_vocab(sentences)\nvocab_size = len(words)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c0c06eb7197a123dd19c2e6418e0bf6c30407dd1"},"cell_type":"markdown","source":"## Make the embedding"},{"metadata":{"trusted":true,"_uuid":"f97cd7ebe5e5f7eb6c28ddc744c7aed14361fc87"},"cell_type":"code","source":"def get_embedding_matrix(words_training):\n    embedding_matrix = np.zeros((len(words_training), 300))\n    mapping = {}\n    for index, word in enumerate(words_training):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[index] = embedding_vector\n            mapping[word] = index\n    return (mapping, embedding_matrix)\n           \nmapping, embedding_matrix = get_embedding_matrix(words)\ninput_sequences = [[mapping[j] for j in i if j in mapping.keys()] for i in sentences]\nvocab_size = len(set([item for sublist in sentences for item in sublist]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4a0df682e0e19c2e7e058335545a33dcdaa5e87"},"cell_type":"markdown","source":"# Assess coverage"},{"metadata":{"trusted":true,"_uuid":"13494c016168e5f0d27787c717e91982760add2b"},"cell_type":"code","source":"print(\"Number of words with embedding:\", len(mapping.keys()))\nprint(\"Percentage of embedded:\", len(mapping.keys())/vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13aaba0a3efa7d533ecb4a5ad38c227d1ebca185"},"cell_type":"markdown","source":"# Prepare train/validation split"},{"metadata":{"trusted":true,"_uuid":"026cbcde1b6bf610aa4351ae1b562adf2460a299"},"cell_type":"code","source":"from math import ceil\n\nlabels = train.target\nhalf = ceil(len(labels)/2)\nprint(\"Size training:\", half)\nprint(\"Insincere train:\", np.sum(labels[0:half]))\nprint(\"Insincere validation:\", np.sum(labels[half+1:len(labels)]))\n\ntrain_sentences = input_sequences[0:half]\nvalidation_sentences = input_sequences[half+1:len(labels)]\ntrain_labels = train.target[0:half]\nvalidation_labels = train.target[half+1:len(labels)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da648c36daf851b563b98d7ab9a6140a7b394588"},"cell_type":"markdown","source":"# Train the model"},{"metadata":{"trusted":true,"_uuid":"c6381c067fda387b47ac4eb72a66de3401905eb8"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import Embedding\nfrom keras.layers import Dropout\nfrom keras.layers import Conv1D\nfrom keras.layers import MaxPooling1D\nfrom keras.layers import LSTM\nfrom keras.preprocessing.sequence import pad_sequences\n\ndef create_model(vocab_size, embedding_size, max_words, embedding_matrix, input_sequences, labels):\n    padded_docs = pad_sequences(input_sequences, maxlen=max_words, padding='post')\n    model = Sequential()\n    model.add(Embedding(vocab_size, embedding_size, input_length=max_words, \n                        weights=[embedding_matrix], trainable=False))   \n    model.add(Dropout(0.2))\n    model.add(Conv1D(64, 5, activation='relu'))\n    model.add(MaxPooling1D(pool_size=4))\n    model.add(LSTM(300))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    model.fit(padded_docs, labels, epochs=10)\n    return model\n\nmodel = create_model(vocab_size, embedding_size, max_words, embedding_matrix, \n                     train_sentences, train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41a39aa34ecdbd58cb778060bec5575a4d4891c7"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ntrainset = pad_sequences(train_sentences, maxlen=max_words, padding='post')\nvalidationset = pad_sequences(validation_sentences, maxlen=max_words, padding='post')\npredictions_train = model.predict(trainset)\npredictions_validation = model.predict(validationset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f51b27bb050b36579d855aa2815ff8e0e2d7312c"},"cell_type":"code","source":"threshold = np.linspace(0, 0.6, 1000)\nscore_train = [f1_score(train_labels, \n                        (predictions_train > t).astype(int)) \n               for t in threshold]\nscore_validation = [f1_score(validation_labels, \n                             (predictions_validation > t).astype(int)) \n                    for t in threshold]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85f34d059ebb4f88f707e5880286173a86599c92"},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nplt.style.use('seaborn-whitegrid')\n\nplt.plot(threshold, score_train, threshold, score_validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f82ddd16a48c4d9617899c2e90b871a39d0ccf0"},"cell_type":"code","source":"print(\"F1 score best:\", np.max(score_validation))\nprint(\"Threshold:\", threshold[np.argmax(score_validation)])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}