{"cells":[{"metadata":{"_cell_guid":"9ad2c916-825e-42d6-8ab1-4f1853b013af","_uuid":"cdf65ac4c40e50d7d59436186211c16311ac2292","collapsed":true,"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport csv\nimport tensorflow as tf\nimport nltk\nimport gc\nfrom gensim.models import Word2Vec\nfrom keras.preprocessing import text, sequence\nfrom sklearn.model_selection import train_test_split\nfrom collections import Counter\nimport math","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4b19e1b4-3749-4629-98af-c6d41a43b423","_uuid":"5e720b768e5215a4b4ed9adfffce57fd61a11b40","collapsed":true,"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/avito-demand-prediction/train.csv') \ntrain_input = df_train['title']\ny_train = df_train['deal_probability']","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"70264abb-1937-41a9-b96d-66813ef159a4","_uuid":"42fff481e721c88ecf52d6efe49ecdc631f031fb","collapsed":true,"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('../input/avito-demand-prediction/test.csv')\ntest_input = df_test['title']","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"44476e2a-0859-4225-90b2-93f5c840acc2","_uuid":"1854aeef6ad50d0752a9b2cd1fe947674c468260","collapsed":true,"trusted":true},"cell_type":"code","source":"del df_train\ndel df_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"87a3966b-9f2e-492b-b5dd-77d3e28f22f7","_uuid":"99f4ce9cec05a31c70cd78b153070f62a84afb88","collapsed":true,"trusted":true},"cell_type":"code","source":"def get_coefs(word, *arr): \n    return word, np.asarray(arr, dtype='float32')\n\nembeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open('../input/fasttext-russian-2m/wiki.ru.vec'))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6ffb2e57-17f5-4aee-bd67-c2ab7bb08fd5","_uuid":"e736e097b8e7e59325aca766b4664d41411ef107","collapsed":true,"trusted":true},"cell_type":"code","source":"len(embeddings_index) ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"649ef7eb-e360-424e-a272-faf44d7088e0","_uuid":"5818642dbc78d7ee7b864561bcb7f7e195c47b62","collapsed":true,"trusted":true},"cell_type":"code","source":"del embeddings_index['1888423']","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"26b695c8-d926-4ecd-8d58-bee336747512","_uuid":"dcc44b83800af8d0161cf5ffd422c5ef7a4d42bf","collapsed":true,"trusted":true},"cell_type":"code","source":"def clean(string):\n    string = re.sub(r'\\n', ' ', string)\n    string = re.sub(r'\\t', ' ', string)\n    string = re.sub('[\\W]', ' ', string)\n    string = re.sub('[0-9]', ' ', string)\n    string = re.sub(r'\\s{2,}', ' ', string.lower())\n    return string","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7750c555-5ec6-423c-a7ee-e696bdcdcbbb","_uuid":"135f72a21b9884f9a6953a5ef61b4967d819cbff","collapsed":true,"trusted":true},"cell_type":"code","source":"x_train = train_input.apply(clean)\nx_test = test_input.apply(clean)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5b601ab1-9d1c-49c5-9c01-c8f5025851b9","_uuid":"4f6192866dae6b4311a106578a8d72543455b7b5","collapsed":true,"trusted":true},"cell_type":"code","source":"x_train = x_train.fillna('fillna')\nx_test = x_test.fillna('fillna')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ff9fa267-67d9-4fd0-8a1a-d7cf96dd1737","_uuid":"6b8965b9ca67591e7e63af895dbc54077feb95b2","collapsed":true,"trusted":true},"cell_type":"code","source":"lst = []\nfor line in x_train:\n    lst += line.split()\n    \ncount = Counter(lst)\nfor k in list(count.keys()):\n    if k not in embeddings_index:\n        del count[k]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e46e039a-9641-4c6e-881b-6095c82fc235","_uuid":"b60db3168a7757bf92c4123e1e3a678f2729658e","collapsed":true,"trusted":true},"cell_type":"code","source":"len(count)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f9913174-c649-4632-a6c4-5046eebe3a23","_uuid":"ac12c0d58bb3d7fb04a08a396a0555be92d82943","collapsed":true,"trusted":true},"cell_type":"code","source":"count = dict(sorted(count.items(), key=lambda x: -x[1]))\ncount = {k:v for (k,v) in count.items() if v >= 3}","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"dadac9e8-be3a-4953-ab0a-8f0ccfc9c424","_uuid":"a83702a865d11e801f9fffda753980e37282cd9a","collapsed":true,"trusted":true},"cell_type":"code","source":"len(count)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f59d0882-fd06-45ba-a594-c0b0c335a020","_uuid":"e93d035ae72d4dc46fdfc21b77f7fe55e192303f","collapsed":true,"trusted":true},"cell_type":"code","source":"count = dict(zip(list(count.keys()),range(1,43547 + 1)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6d4a2177-fbd6-41ff-89ab-c01e6f73b083","_uuid":"4db4d7c203495246ed2cbe54e2567f715a1f070d","collapsed":true,"trusted":true},"cell_type":"code","source":"embedding_matrix = {}\nfor key in count:\n    embedding_matrix[key] = embeddings_index[key]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c5dcdeb1-8a41-435d-aa3f-a60979e0d3ac","_uuid":"05c6d2f23e29f3c6ab1a782ef6c1f62f33e8102a","collapsed":true,"trusted":true},"cell_type":"code","source":"lst = []\nfor line in x_test:\n    lst += line.split()\n    \ncount_test = Counter(lst)\nfor k in list(count_test.keys()):\n    if k not in embedding_matrix:\n        del count_test[k]\n    else:\n        count_test[k] = count[k]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ece913e9-6178-44c5-83a6-8a6d0e470dd7","_uuid":"25b4a9572283a194247e1f9e8dacd28216cc3be4","collapsed":true,"trusted":true},"cell_type":"code","source":"len(count_test)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"86ede5cc-8e55-4398-b3c4-fdc0ac88a1c4","_uuid":"7d5ca629e84a42b8f1cee1abc287b3bae72b1db5","collapsed":true,"trusted":true},"cell_type":"code","source":"W = np.zeros((1,300))\nW = np.append(W, np.array(list(embedding_matrix.values())),axis=0)\nW = W.astype(np.float32, copy=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cf6d985b-c5db-4cfe-8651-c3a3bee5a2b3","_uuid":"8866e6d5dfdef0df4bb6507e2a682473ce0cee6f","collapsed":true,"trusted":true},"cell_type":"code","source":"del lst\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b7e75e59-6b0a-4563-9820-5b57817cd25c","_uuid":"e36da6d1164e0b745d0accb37fe3f24d107d6c64","collapsed":true,"trusted":true},"cell_type":"code","source":"for i in range(len(x_train)):\n    temp = x_train[i].split()\n    for word in temp[:]:\n        if word not in count:\n            temp.remove(word)\n    for j in range(len(temp)):\n        temp[j] = count[temp[j]]\n    x_train[i] = temp\n    \nfor i in range(len(x_test)):\n    temp = x_test[i].split()\n    for word in temp[:]:\n        if word not in count_test:\n            temp.remove(word)\n    for j in range(len(temp)):\n        temp[j] = count_test[temp[j]]\n    x_test[i] = temp","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c4ca031f-0031-47ed-b2ae-3e54d1d1c606","_uuid":"6b14e7aa4888438f71ab3f80595cbc62ea6d6456","collapsed":true,"trusted":true},"cell_type":"code","source":"x_train = sequence.pad_sequences(list(x_train), maxlen = 15)\nx_test = sequence.pad_sequences(list(x_test), maxlen = 15)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3dcf1088-f575-4c7f-adc6-75c953f965d5","_uuid":"7ba6f34d4ce32b088373916e17b0ecbbb6f50da4"},"cell_type":"markdown","source":"# Build CNN"},{"metadata":{"_cell_guid":"f7b8c6c0-a7da-4e8d-9603-a4f79ebc2741","_uuid":"7d3b569890ca22cd6be907f5972269fa9b4302ec","collapsed":true,"trusted":true},"cell_type":"code","source":"filter_sizes = [1,2,3,4,5]\nnum_filters = 32\nbatch_size = 256\nnum_filters_total = num_filters * len(filter_sizes)\nembedding_size = 300\nsequence_length = 15\nnum_epochs = 3\ndropout_keep_prob = 0.8","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1aa7c46c-21e8-4bfe-900c-ae2d6ab4c272","_uuid":"69666c0010ae93e1621e6d415827ae61311fb261","collapsed":true,"trusted":true},"cell_type":"code","source":"input_x = tf.placeholder(tf.int32, [None, sequence_length], name = \"input_x\")\ninput_y = tf.placeholder(tf.float32, [None,2], name = \"input_y\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c76aae85-86ba-477a-92bc-25427aaf955f","_uuid":"9354e87e5cf2509387f68945524e5a14e85156a4","collapsed":true,"trusted":true},"cell_type":"code","source":"embedded_chars = tf.nn.embedding_lookup(W, input_x)\nembedded_chars_expanded = tf.expand_dims(embedded_chars, -1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a3220b02-f1d1-4368-9245-3a5d9062db9e","_uuid":"cde891af56603fd9a5bbb4d66e3a2aa5392d900e","collapsed":true,"trusted":true},"cell_type":"code","source":"def CNN(data):\n    pooled_outputs = []\n    \n    for i, filter_size in enumerate(filter_sizes):\n        \n        filter_shape = [filter_size, embedding_size, 1, num_filters]\n        \n        w = tf.Variable(tf.truncated_normal(filter_shape,stddev = 0.05), name = \"w\")\n        b = tf.Variable(tf.truncated_normal([num_filters], stddev = 0.05), name = \"b\")\n            \n        conv = tf.nn.conv2d(\n            data,\n            w,\n            strides = [1,1,1,1],\n            padding = \"VALID\",\n            name = \"conv\"\n        )\n        h = tf.nn.relu(tf.nn.bias_add(conv, b), name = \"relu\")\n        pooled = tf.nn.max_pool(\n            h,\n            ksize = [1, sequence_length - filter_size + 1, 1, 1],\n            strides = [1,1,1,1],\n            padding = \"VALID\",\n            name = \"pool\"\n        )\n        \n        pooled_outputs.append(pooled)\n    \n    #return pooled_outputs\n    h_pool = tf.concat(pooled_outputs, 3)\n    h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total])\n    return h_pool_flat","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c08b2849-e96c-4e0c-aed7-b24ad8943e85","_uuid":"19bf9b57af760329cddb178b82af70b22d467b02","collapsed":true,"trusted":true},"cell_type":"code","source":"h_pool_flat = CNN(embedded_chars_expanded)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4eabbb9b-cb6b-444b-aa02-8ba396e251fd","_uuid":"3d42a2a49cbb6bd6acf59bd0077bec10f1ef98c8","collapsed":true,"trusted":true},"cell_type":"code","source":"h_drop = tf.nn.dropout(h_pool_flat, dropout_keep_prob)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"da5c0336-c541-4e16-a820-1ad8aef04fbd","_uuid":"761ea720cae639b751ffe05ccd95387710890cde","collapsed":true,"trusted":true},"cell_type":"code","source":"wd1 = tf.Variable(tf.truncated_normal([num_filters_total, int(num_filters_total/2)], stddev=0.05), name = \"wd1\")\nbd1 = tf.Variable(tf.truncated_normal([int(num_filters_total/2)], stddev = 0.05), name = \"bd1\")\nlayer1 = tf.nn.xw_plus_b(h_drop, wd1, bd1, name = 'layer1')\nlayer1 = tf.nn.relu(layer1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fd9c5dc9-6ae9-4c8d-ade8-263780ac03f9","_uuid":"ed4a7aae6e0d31b81d816b77a83c226c5282c27b","collapsed":true,"trusted":true},"cell_type":"code","source":"wd2 = tf.Variable(tf.truncated_normal([int(num_filters_total/2),2], stddev = 0.05), name = 'wd2')\nbd2 = tf.Variable(tf.truncated_normal([2], stddev = 0.05), name = \"bd2\")\nlayer2 = tf.nn.xw_plus_b(layer1, wd2, bd2, name = 'layer2') \nprediction = tf.nn.softmax(layer2)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"eebea741-f821-477b-8332-d5399e975c89","_uuid":"6015be806460cb7c36fcedc977f4e809b1d06112","collapsed":true,"trusted":true},"cell_type":"code","source":"rmse = tf.reduce_mean(tf.losses.mean_squared_error(predictions = prediction, labels = input_y))\noptimizer = tf.train.AdamOptimizer(learning_rate = 0.0007).minimize(rmse)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f5a67d01-0e60-4e88-9b60-e9b18610fd26","_uuid":"a364fbc72079f966aa4b0526052e2ff241678f64","collapsed":true,"trusted":true},"cell_type":"code","source":"def generate_batch(data, batch_size, num_epochs, shuffle=True):\n    data = np.array(data)\n    data_size = len(data)\n    num_batches_per_epoch = int((len(data)-1)/batch_size) + 1\n    l = 0\n    for epoch in range(num_epochs):\n        l += 1\n        if shuffle:\n            shuffle_indices = np.random.permutation(np.arange(data_size))\n            shuffled_data = data[shuffle_indices]\n        else:\n            shuffled_data = data\n        for batch_num in range(num_batches_per_epoch):\n            start_index = batch_num * batch_size\n            end_index = min((batch_num + 1) * batch_size, data_size)\n            yield shuffled_data[start_index:end_index]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"df6225c0-0dcd-48e6-b5ed-89dd51c0d8df","_uuid":"31c6a91bdf3d8725b65a5fa8162188e4a15e2dfc","collapsed":true,"trusted":true},"cell_type":"code","source":"def blocks(data, block_size):\n    data = np.array(data)\n    data_size = len(data)\n    nums = int((data_size-1)/block_size) + 1\n    for block_num in range(nums):\n        if block_num == 0:\n            print(\"prediction start!\")\n        start_index = block_num * block_size\n        end_index = min((block_num + 1) * block_size, data_size)\n        yield data[start_index:end_index]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8ec77586-e6f6-450f-b48b-7c34d922b058","_uuid":"8836cd9526f1b9b11f103bc3825c20dcaa43a79a","collapsed":true,"trusted":true},"cell_type":"code","source":"batch1 = generate_batch(list(zip(np.array(x_train), y_train)), batch_size, 1)\nbatch2 = generate_batch(list(zip(np.array(x_train), y_train)), batch_size, 1)\nbatch3 = generate_batch(list(zip(np.array(x_train), y_train)), batch_size, 1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"46d98709-3ef7-4507-81d6-723e93bb28b2","_uuid":"1fc1f64573a8e6295d0d8e427948817d8dd0dc7f","collapsed":true,"trusted":true},"cell_type":"code","source":"batch_bag = [batch1,batch2,batch3]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b70766cc-d623-4543-b326-b42238e789eb","_uuid":"0d075a3d1b0dada8fcc7b0f37889314d5d6d6ea0","collapsed":true,"trusted":true},"cell_type":"code","source":"test_blocks = blocks(list(np.array(x_test)), 2000)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4d6dcee9-4537-4fd8-b201-862c3a666b71","_uuid":"09344713fbcc43648de7195b4a56552d89fcb225","collapsed":true,"trusted":true},"cell_type":"code","source":"int((len(x_train)-1)/256) + 1","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b00152e8-90ba-461b-a8af-4c8967fd0f4c","_uuid":"ab6c7b372e4be5951ff2ec74fb50598920d98caa","collapsed":true,"scrolled":true,"trusted":true},"cell_type":"code","source":"init_op = tf.global_variables_initializer()\n\nwith tf.Session() as sess:\n    \n    sess.run(init_op)\n    i = 0\n    for batches in batch_bag:\n        i += 1\n        print('Epoch: ' + str(i) + ' start!')\n        avg_loss = 0\n        avg_rmse = 0\n        for batch in batches:\n            batch = pd.DataFrame(batch, columns = ['a','1'])\n            x_batch = pd.DataFrame(list(batch['a']))\n            y_batch = batch.loc[:, batch.columns != 'a']\n            y_batch['0'] = 1 - y_batch['1']\n            _,m = sess.run([optimizer, rmse],feed_dict = {input_x: x_batch, input_y: y_batch})\n            avg_rmse += m\n        avg_rmse = math.sqrt(avg_rmse/5873)\n        print('Epoch:' + str(i) + ' Rmse is ' + str(avg_rmse))\n        \n    print('Prediction Start!')\n    \n    df = pd.DataFrame()\n    for block in test_blocks:\n        block = pd.DataFrame(block)\n        pred = sess.run(prediction, feed_dict = {input_x: block})\n        df = df.append(pd.DataFrame(pred))\n    \n    print('Finish!') ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"606e3d30-0835-45b7-9e9a-5faabb3230ae","_uuid":"0427352d0fd6a8c0c579d746ce78d62c7bfd7386","collapsed":true,"trusted":true},"cell_type":"code","source":"df.round().mean()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3e7abe0a-0944-4850-9d2d-a7b1db0b2b32","_uuid":"6442542782b27d3aafec1410fa8f428981e4a769","collapsed":true,"trusted":true},"cell_type":"code","source":"df.columns = ['0','1']","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"38ac8f70-bf62-45f8-af20-71ad5833d8fa","_uuid":"afb851eedde9c8b58ed13809fd3986a67ae1efb1","collapsed":true,"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"../input/avito-demand-prediction/sample_submission.csv\")\nsubmission['deal_probability'] = np.array(df['0'])\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}