{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"cell_type":"markdown","source":"\"drawing\""},{"metadata":{"_cell_guid":"e4c68459-1fe8-4f82-b378-19b55c9f18c4","_uuid":"3676c45a01aab90fe2ebb27051dc826b164424fb"},"cell_type":"markdown","source":"image from: \n- https://en.wikipedia.org/wiki/Russian_cuisine\n\n## Mixed Arch NN Recipe\n\n- FM-like: https://www.kaggle.com/qqgeogor/keras-based-fm\n- Attention: https://www.kaggle.com/sermakarevich/hierarchical-attention-network\n- RNN: https://www.kaggle.com/yekenot/pooled-gru-fasttext\n- CNN\n- FC\n\n## Note\n- Here just try for different nn archs and it is really **NOT necessary** to combine everything togather\n- Just like each other ML parts , the best features/archs/models still need to be decided by various experiments"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nDATA_DIR = '../input/avito-demand-prediction/'\nEMB_PATH = '../input/fasttest-common-crawl-russian/cc.ru.300.vec'\n#EMB_PATH = '../input/fasttext-russian-2m/wiki.ru.vec'\n#EMB_PATH = '../input/russian-glove/multilingual_embeddings.ru'\ntarget_col = 'deal_probability'\nmax_features = 50000\nmaxlen = 100\nembed_size = 300\nos.listdir(DATA_DIR)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a1b4be61-c793-46f1-a5da-e2c1499f08f5","collapsed":true,"_uuid":"123a389d5a2662ca837b821df750069c1a11dd4d","trusted":false},"cell_type":"code","source":"usecols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1',\n 'param_2', 'param_3', 'title', 'description', 'price', 'activation_date',\n 'item_seq_number', 'user_type', 'image_top_1']\ntrain = pd.read_csv(DATA_DIR+'train.csv', usecols=usecols+[target_col])\ntest = pd.read_csv(DATA_DIR+'test.csv', usecols=usecols)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"399922a6-fb97-43b0-8954-72a4dfed469f","collapsed":true,"_uuid":"539cb751b0f8c482ba6e23ee7adf18b74af904c6","trusted":false},"cell_type":"code","source":"train = train[train['activation_date']<'2017-03-29']\ntest.loc[test['activation_date']>'2017-04-18', 'activation_date'] = '2017-04-18'","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8616c75d-e8d4-40a1-997e-698328e82032","_uuid":"b8bc362ea09f2568f48b6f4939ef400519bf9896","trusted":false,"collapsed":true},"cell_type":"code","source":"train['activation_date'].min(), train['activation_date'].max()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3a45b767-d831-4746-8442-bd709bad1859","_uuid":"4ab24582e4ae528c52e946df5e7ec08b45dd3bb0","trusted":false,"collapsed":true},"cell_type":"code","source":"test['activation_date'].min(), test['activation_date'].max()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"03874d3d-084c-4ef9-b6e4-d133def8b42f","_uuid":"f2dea66b285892793b4c0949e65921f5c7a18880","trusted":false,"collapsed":true},"cell_type":"code","source":"%%time\ntrain['na_cnt'] = train.isnull().sum(axis=1)\ntrain['na_price'] = train['price'].isnull().astype('int8')\ntest['na_cnt'] = test.isnull().sum(axis=1)\ntest['na_price'] = test['price'].isnull().astype('int8')\ntrain['item_seq_number_log1p'] = train['item_seq_number'].apply(lambda x:np.log1p(x))\ntest['item_seq_number_log1p'] = test['item_seq_number'].apply(lambda x:np.log1p(x))\ntrain['image_top_1'] = train['image_top_1'].fillna(train['image_top_1'].max()+1)\ntest['image_top_1'] = test['image_top_1'].fillna(test['image_top_1'].max()+1)\n\nsize_cols = ['region', 'city', 'parent_category_name', 'category_name', 'image_top_1']\nmean_cols = ['parent_category_name', 'category_name', 'image_top_1']\n\ndef add_group_size(df, by, y='price'):\n grp = df.groupby(by)[y].size().map(lambda x:np.log1p(x))\n grp = grp.rename('size_'+'_'.join(by)).reset_index()\n df = df.merge(grp, on=by, how='left')\n return df\n\ndef add_group_mean(df, by, y='price'):\n grp = df.groupby(by)[y].mean()\n grp = grp.rename('mean_price_'+'_'.join(by)).reset_index()\n df = df.merge(grp, on=by, how='left')\n return df\n\nfor c in size_cols:\n print('adding size by', [c, 'activation_date'])\n train = add_group_size(train, [c, 'activation_date'])\n test = add_group_size(test, [c, 'activation_date'])\n \nfor c in mean_cols:\n print('adding mean price by', (c,))\n train = add_group_mean(train, (c,))\n test = add_group_mean(test, (c,))\n \ndel train['activation_date'], test['activation_date']; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"315d9aa4-daf6-4183-bde0-5be987d35166","_uuid":"9f4b4d62f926a5d3c83dda803704fe6005c9901b","trusted":false,"collapsed":true},"cell_type":"code","source":"%%time\nsize_cols = [c for c in train.columns if 'size_' in c]\nmean_cols = [c for c in train.columns if 'mean_' in c]\n\nfor c in ['price'] + mean_cols:\n train[c] = train[c].fillna(-1)#-1\n test[c] = test[c].fillna(-1)#-1\n train[c] = train[c].apply(lambda x:np.log1p(x))\n test[c] = test[c].apply(lambda x:np.log1p(x))\n\ntrain = train.fillna('неизвестный')\ntest = test.fillna('неизвестный')\ny = train[target_col].values\ndel train[target_col]; gc.collect()\ntrain_num = len(train)\n\nstr_cols = [c for c in train.columns if c not in [\n 'na_price', 'na_cnt', 'item_seq_number_log1p',\n 'price', 'item_seq_number', 'image_top_1', 'user_type', target_col] + size_cols + mean_cols\n ]\n\ncat_cols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', \n 'item_seq_number', 'user_type', 'image_top_1']\n\ndf = pd.concat([train, test], ignore_index=True)\ndel train, test; gc.collect()\n\ndf['text'] = ''\nfor i, c in enumerate(df.columns):\n if c in str_cols:\n df['text'] += ' ' + df[c].str.lower()\n if c not in cat_cols:\n del df[c]; gc.collect()\n if c in cat_cols:\n df[c] = pd.factorize(df[c])[0]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"adaab70c-37dc-4c14-9cfb-74f9b36f58c3","_uuid":"9485d7fa9461c9639e89e260995a182cf593d992","trusted":false,"collapsed":true},"cell_type":"code","source":"for c in cat_cols:\n print(c, df[c].min(), df[c].max())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b15665bd-c38e-4e45-ba0a-fe9ad347b1eb","_uuid":"1b22eb2d2bbcff1a617563f7da154ff7e28dca3f","trusted":false,"collapsed":true},"cell_type":"code","source":"print(df.info())\nfor c in cat_cols+['na_cnt']:\n if df[c].max()<2**7:\n df[c] = df[c].astype('int8')\n elif df[c].max()<2**15:\n df[c] = df[c].astype('int16')\n elif df[c].max()<2**31:\n df[c] = df[c].astype('int32')\n else:\n continue\nprint(df.info())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"41155b84-45b3-4a90-a20c-c29bacb8246b","collapsed":true,"_uuid":"61044949e646f388b292487d6c3aa8b9c1fa76c2","trusted":false},"cell_type":"code","source":"df['char_len'] = df['text'].apply(lambda x:np.log1p(len(x)))\ndf['char_len'] = df['char_len'].astype('float32')\ndf['word_len'] = df['text'].apply(lambda x:np.log1p(len(x.split(' '))))\ndf['word_len'] = df['word_len'].astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"03ebd957-8b8c-4735-85cc-976e56cab0a7","_uuid":"f1cd18b3b56bc3794490ca5a1a8ba5e577d6d343","trusted":false,"collapsed":true},"cell_type":"code","source":"df['char_len'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6a93e166-c0fd-4ad9-a43c-84ddc74c1fb8","_uuid":"fbe3866dad15a43a6e7d7bd73d611bc4b056a3dc","trusted":false,"collapsed":true},"cell_type":"code","source":"df['word_len'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b012a7ff-df0a-4189-8a69-df75bb03e488","_uuid":"67b71d7431e4cb2c8118f2d5ad2e0e61e75bc390","trusted":false,"collapsed":true},"cell_type":"code","source":"df.head(3).T","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f4e88f50-e786-4636-b0b6-8e8b4d42adbd","_uuid":"8afe1cdf7dfbe971efea3be9be88b01d68eaea2e","trusted":false,"collapsed":true},"cell_type":"code","source":"df.describe().T","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b3b16c39-2fc0-41d1-af8e-be1701f39668","_uuid":"78cd73203089d4bd10581d5e4690093a0cdc766f","trusted":false,"collapsed":true},"cell_type":"code","source":"%%time\nfrom keras.preprocessing import text, sequence\nprint('tokenizing...')\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(df['text'].values.tolist())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9aa6470e-ff7a-4e93-ad10-2185f81c8ca7","_uuid":"5a3f6bc14c31b99f8b121c89fe062c3a5563a6cf","trusted":false,"collapsed":true},"cell_type":"code","source":"%%time\nprint('getting embeddings')\ndef get_coefs(word, *arr, tokenizer=None):\n if tokenizer is None:\n return word, np.asarray(arr, dtype='float32')\n else:\n if word not in tokenizer.word_index:\n return None\n else:\n return word, np.asarray(arr, dtype='float32')\nnb_words = min(max_features, len(tokenizer.word_index))\nembedding_matrix = np.zeros((nb_words, embed_size))\nfor o in tqdm(open(EMB_PATH)):\n res = get_coefs(*o.rstrip().rsplit(' '), tokenizer=tokenizer)\n if res is not None:\n idx = tokenizer.word_index[res[0]]\n if idx < max_features:\n embedding_matrix[idx] = res[1]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0ebb1c65-120c-490f-b976-022a31a0f56d","_uuid":"14b761939299fc737ef63f26c8e1063436da7803","trusted":false,"collapsed":true},"cell_type":"code","source":"%%time\ndef fill_rand_norm(embedding_matrix):\n emb_zero_shape = embedding_matrix[embedding_matrix==0].shape\n emb_non_zero_mean = embedding_matrix[embedding_matrix!=0.].mean()\n emb_non_zero_std = embedding_matrix[embedding_matrix!=0.].std()\n embedding_matrix[embedding_matrix==0] = np.random.normal(emb_non_zero_mean, \n emb_non_zero_std, \n emb_zero_shape)\n return embedding_matrix\nembedding_matrix = fill_rand_norm(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c12668a5-cc70-4f0c-b6fa-46f799b346cd","_uuid":"570b716f52a9df74839dc66db28866d7a6e337de","trusted":false,"collapsed":true},"cell_type":"code","source":"%%time\ntext = df['text'].values\ndel df['text']; gc.collect()\ntext = tokenizer.texts_to_sequences(text)\ntext = sequence.pad_sequences(text, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"381e8924-ec49-4285-8201-eb5863ef01de","_uuid":"24542f6a3d9b2a0dfefdaccd90f241c13f33041f","trusted":false,"collapsed":true},"cell_type":"code","source":"%%time\ndf_train = df[:train_num]\ntext_train = text[:train_num]\ndf_test = df[train_num:]\ntext_test = text[train_num:]\ndel text, df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ea9d800c-70e6-4d97-8c8a-724a03b1bf74","_uuid":"7417144ecbffdd9a32c433ce86699b6fb3806169","trusted":false,"collapsed":true},"cell_type":"code","source":"%%time\ndef get_keras_data(df, text):\n X = {}\n for c in df.columns:\n X[c] = df[c].values\n X['text'] = text\n return X\nX_train = get_keras_data(df_train, text_train)\nX_test = get_keras_data(df_test, text_test)\ndel df_train, text_train, df_test, text_test; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"99c1e82d-f496-479d-a339-f0b28f2936af","collapsed":true,"_uuid":"50f4b1060704b40fd31ddb3cdc1421a26498d603","trusted":false},"cell_type":"code","source":"import keras.backend as K\nfrom keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate\nfrom keras.layers import \\\n BatchNormalization, SpatialDropout1D, GlobalAveragePooling1D, GRU, Bidirectional, GlobalMaxPooling1D, Conv1D\nfrom keras.layers import CuDNNGRU\nfrom keras.layers import add, dot\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom itertools import combinations\n\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints\n\n\"\"\"\nfrom: https://www.kaggle.com/sermakarevich/hierarchical-attention-network\n\"\"\"\ndef dot_product(x, kernel):\n \"\"\"\n Wrapper for dot product operation, in order to be compatible with both\n Theano and Tensorflow\n Args:\n x (): input\n kernel (): weights\n Returns:\n \"\"\"\n if K.backend() == 'tensorflow':\n return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)\n else:\n return K.dot(x, kernel)\n \nclass AttentionWithContext(Layer):\n\n \"\"\"\n Attention operation, with a context/query vector, for temporal data.\n Supports Masking.\n Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]\n \"Hierarchical Attention Networks for Document Classification\"\n by using a context vector to assist the attention\n # Input shape\n 3D tensor with shape: `(samples, steps, features)`.\n # Output shape\n 2D tensor with shape: `(samples, features)`.\n How to use:\n Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n The dimensions are inferred based on the output shape of the RNN.\n Note: The layer has been tested with Keras 2.0.6\n Example:\n model.add(LSTM(64, return_sequences=True))\n model.add(AttentionWithContext())\n # next add a Dense layer (for classification/regression) or whatever...\n \"\"\"\n def __init__(self,\n W_regularizer=None, u_regularizer=None, b_regularizer=None,\n W_constraint=None, u_constraint=None, b_constraint=None,\n bias=True, **kwargs):\n\n self.supports_masking = True\n self.init = initializers.get('glorot_uniform')\n\n self.W_regularizer = regularizers.get(W_regularizer)\n self.u_regularizer = regularizers.get(u_regularizer)\n self.b_regularizer = regularizers.get(b_regularizer)\n\n self.W_constraint = constraints.get(W_constraint)\n self.u_constraint = constraints.get(u_constraint)\n self.b_constraint = constraints.get(b_constraint)\n\n self.bias = bias\n super(AttentionWithContext, self).__init__(**kwargs)\n\n def build(self, input_shape):\n assert len(input_shape) == 3\n\n self.W = self.add_weight((input_shape[-1], input_shape[-1],),\n initializer=self.init,\n name='{}_W'.format(self.name),\n regularizer=self.W_regularizer,\n constraint=self.W_constraint)\n if self.bias:\n self.b = self.add_weight((input_shape[-1],),\n initializer='zero',\n name='{}_b'.format(self.name),\n regularizer=self.b_regularizer,\n constraint=self.b_constraint)\n\n self.u = self.add_weight((input_shape[-1],),\n initializer=self.init,\n name='{}_u'.format(self.name),\n regularizer=self.u_regularizer,\n constraint=self.u_constraint)\n\n super(AttentionWithContext, self).build(input_shape)\n\n def compute_mask(self, input, input_mask=None):\n # do not pass the mask to the next layers\n return None\n\n def call(self, x, mask=None):\n uit = dot_product(x, self.W)\n\n if self.bias:\n uit += self.b\n\n uit = K.tanh(uit)\n ait = dot_product(uit, self.u)\n\n a = K.exp(ait)\n\n # apply mask after the exp. will be re-normalized next\n if mask is not None:\n # Cast the mask to floatX to avoid float64 upcasting in theano\n a *= K.cast(mask, K.floatx())\n\n # in some cases especially in the early stages of training the sum may be almost zero\n # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.\n # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())\n a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n a = K.expand_dims(a)\n weighted_input = x * a\n return K.sum(weighted_input, axis=1)\n\n def compute_output_shape(self, input_shape):\n return input_shape[0], input_shape[-1]\n ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d258f29c-fd54-41a9-b0eb-25b38d7462f0","collapsed":true,"_uuid":"e261bd907e924e5852fa3a255ca3b738f66a6a2b","trusted":false},"cell_type":"code","source":"emb_cat_max = {}\nfor c in cat_cols:\n emb_cat_max[c] = max(X_train[c].max(), X_test[c].max())+1\nparams = {}\nparams['maxlen'] = maxlen\nparams['nb_words'] = nb_words\nparams['embedding_matrix'] = embedding_matrix\nparams['word_emb_size'] = embed_size\nparams['text_emb_dropout'] = 0.2\nparams['n_rnn'] = 64\n\nparams['emb_cat_max'] = emb_cat_max\nparams['emb_size'] = 32\nparams['n_output'] = 1\nparams['use_fm'] = True\nparams['use_deep'] = True\nparams['use_rnn'] = True\nparams['use_cnn'] = True\nparams['use_att'] = False\nparams['use_batch_norm'] = False #replace dropout \"\"\"loss: inf ???\"\"\"\nparams['cnn_param'] = dict(filters=192, kernel_size=3)\nparams['deep_layers'] = [256, 256, 256]\nparams['drop_out'] = [0.5, 0.5, 0.5]\nparams['output_drop_out'] = 0.2\nassert len(params['drop_out'])==len(params['deep_layers'])\nparams['cat_feats'] = cat_cols\nparams['num_feats'] = size_cols+mean_cols+['price', 'char_len', 'word_len', 'na_price', 'na_cnt', 'item_seq_number_log1p']\nparams['lr'] = 0.001\nparams['decay'] = 1e-6","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"401e28a6-f91e-485e-8238-32b90c0f7849","_uuid":"a41cc5d224bfad9a7fb3d0d10c5bb8f32c0b7c89","trusted":false,"collapsed":true},"cell_type":"code","source":"def root_mean_squared_error(y_true, y_pred):\n return K.sqrt(K.mean(K.square(y_pred - y_true))) \n\n\"\"\"\nMIXED ARCH NN\n\"\"\"\ndef get_model(params=params):\n cats = [Input(shape=[1], name=name) for name in params['cat_feats']]\n nums = [Input(shape=[1], name=name) for name in params['num_feats']]\n emb_fn = lambda name: Embedding(params['emb_cat_max'][name], params['emb_size'])\n embs = []\n for name, cat in zip(params['cat_feats'], cats):\n embs.append(emb_fn(name)(cat))\n \n texts = Input(shape=(params['maxlen'],), name='text')\n text_emb = Embedding(params['nb_words'], \n params['word_emb_size'], \n weights=[params['embedding_matrix']],\n name='text_emb')(texts)\n \n outs = []\n if params['use_rnn']:\n x_text = SpatialDropout1D(params['text_emb_dropout'])(text_emb)\n x_rnn = Bidirectional(GRU(params['n_rnn'], return_sequences=True))(x_text)\n avg_pool_rnn = GlobalAveragePooling1D()(x_rnn)\n max_pool_rnn = GlobalMaxPooling1D()(x_rnn)\n outs += [avg_pool_rnn, max_pool_rnn]\n if params['use_cnn']:\n x_text = SpatialDropout1D(params['text_emb_dropout'])(text_emb)\n x_cnn = Conv1D(**params['cnn_param'])(x_text)\n avg_pool_cnn = GlobalAveragePooling1D()(x_cnn)\n max_pool_cnn = GlobalMaxPooling1D()(x_cnn)\n outs += [avg_pool_cnn, max_pool_cnn]\n if params['use_att']:\n x_text = SpatialDropout1D(params['text_emb_dropout'])(text_emb)\n x_att = AttentionWithContext()(x_text)\n outs += [x_att]\n if params['use_rnn']:\n x_rnn_att = AttentionWithContext()(x_rnn)\n outs += [x_rnn_att]\n if params['use_cnn']:\n x_cnn_att = AttentionWithContext()(x_cnn)\n outs += [x_cnn_att]\n if params['use_fm']:\n first_order = [Flatten()(emb0) for emb0 in embs]\n second_order = []\n for emb1, emb2 in combinations(embs, 2):\n dot_layer = dot([Flatten()(emb1), Flatten()(emb2)], axes=1)\n second_order.append(dot_layer)\n first_order = add(first_order)\n second_order = add(second_order)\n outs += [first_order, second_order]\n if params['use_deep']:\n all_in = [Flatten()(emb) for emb in embs] + nums + [Flatten()(text_emb)]\n x_in = concatenate(all_in)\n for idx, (drop_p, num_dense) in enumerate(zip(params['drop_out'], params['deep_layers'])):\n x_in = Dense(num_dense, activation='relu')(x_in)\n if params['use_batch_norm']:\n x_in = (BatchNormalization())(x_in)\n else:\n x_in = Dropout(drop_p)(x_in)\n deep = x_in\n outs += [deep]\n \n total_out = concatenate(outs) if len(outs)>1 else outs[0]\n \n if 0