{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install livelossplot --upgrade --quiet","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-23T14:14:21.375659Z","iopub.execute_input":"2021-08-23T14:14:21.376043Z","iopub.status.idle":"2021-08-23T14:14:21.392756Z","shell.execute_reply.started":"2021-08-23T14:14:21.375949Z","shell.execute_reply":"2021-08-23T14:14:21.391862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pickle\ncolor = sns.color_palette()\n%matplotlib inline\nimport gzip\nimport shutil\nimport re\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom sklearn.preprocessing import Normalizer, MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer\nfrom tensorflow.keras.preprocessing.text import Tokenizer, one_hot\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.layers import Input, Dense, Flatten, concatenate, Embedding, LSTM,GRU, Dropout, BatchNormalization, Conv2D, MaxPooling2D, Conv1D, MaxPooling1D\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TerminateOnNaN\nfrom tensorflow.keras.optimizers import Adam, RMSprop\nfrom tensorflow.keras.regularizers import l2, l1\nfrom tensorflow.keras.applications import InceptionResNetV2","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:14:22.319566Z","iopub.execute_input":"2021-08-23T14:14:22.319882Z","iopub.status.idle":"2021-08-23T14:14:27.47869Z","shell.execute_reply.started":"2021-08-23T14:14:22.319854Z","shell.execute_reply":"2021-08-23T14:14:27.477839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/avito-demand-prediction/train.csv\", parse_dates=[\"activation_date\"])\ntest_df = pd.read_csv(\"/kaggle/input/avito-demand-prediction/test.csv\", parse_dates=[\"activation_date\"])\nprint(\"Train file rows and columns are : \", train_df.shape)\nprint(\"Test file rows and columns are : \", test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:14:27.480168Z","iopub.execute_input":"2021-08-23T14:14:27.480532Z","iopub.status.idle":"2021-08-23T14:15:05.728651Z","shell.execute_reply.started":"2021-08-23T14:14:27.480499Z","shell.execute_reply":"2021-08-23T14:15:05.727717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:05.73037Z","iopub.execute_input":"2021-08-23T14:15:05.730879Z","iopub.status.idle":"2021-08-23T14:15:05.761899Z","shell.execute_reply.started":"2021-08-23T14:15:05.730837Z","shell.execute_reply":"2021-08-23T14:15:05.760951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_name = train_df.columns\nprint('Checking the Number of NaN values per column:')\nfor col in column_name[:-1]:\n    print(f'train {col} has {train_df[col].isnull().sum()} test {col} has {test_df[col].isnull().sum()}')","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:05.763588Z","iopub.execute_input":"2021-08-23T14:15:05.763925Z","iopub.status.idle":"2021-08-23T14:15:08.16407Z","shell.execute_reply.started":"2021-08-23T14:15:05.763891Z","shell.execute_reply":"2021-08-23T14:15:08.162346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# filling the nan value of price with mean value w.r.t its category\ntrain_df['price'] = train_df['price'].fillna(train_df.groupby('category_name')['price'].transform('mean'))\ntest_df['price'] = test_df['price'].fillna(test_df.groupby('category_name')['price'].transform('mean'))","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:08.16544Z","iopub.execute_input":"2021-08-23T14:15:08.165814Z","iopub.status.idle":"2021-08-23T14:15:08.421346Z","shell.execute_reply.started":"2021-08-23T14:15:08.165774Z","shell.execute_reply":"2021-08-23T14:15:08.420471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image'] = train_df['image'].fillna(train_df.groupby('parent_category_name')['image'].apply(lambda x: x.fillna(x.mode()[0])))\ntest_df['image'] =  test_df['image'].fillna(test_df.groupby('parent_category_name')['image'].apply(lambda x: x.fillna(x.mode()[0])))","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:08.422636Z","iopub.execute_input":"2021-08-23T14:15:08.422981Z","iopub.status.idle":"2021-08-23T14:15:14.114115Z","shell.execute_reply.started":"2021-08-23T14:15:08.422944Z","shell.execute_reply":"2021-08-23T14:15:14.113238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# filling NaN values with missing\nfor col in column_name[:-1]:\n    train_df[col] = train_df[col].fillna('missing')\n    test_df[col] = test_df[col].fillna('missing')","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:14.115343Z","iopub.execute_input":"2021-08-23T14:15:14.115704Z","iopub.status.idle":"2021-08-23T14:15:17.157456Z","shell.execute_reply.started":"2021-08-23T14:15:14.115669Z","shell.execute_reply":"2021-08-23T14:15:17.156598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_name = train_df.columns\nprint('Checking the Number of NaN values per column:')\nfor col in column_name:\n    print(f'{col} has {train_df[col].isnull().sum()} {col} has {test_df[col].isnull().sum()}')","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:17.159467Z","iopub.execute_input":"2021-08-23T14:15:17.15984Z","iopub.status.idle":"2021-08-23T14:15:20.065116Z","shell.execute_reply.started":"2021-08-23T14:15:17.159806Z","shell.execute_reply":"2021-08-23T14:15:20.063305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# splitting the data with train test split, I am making the test size = 10% only for getting more data to train\nX = train_df.drop(['deal_probability'], axis=1)\nY = train_df['deal_probability']\nX_train, X_cv, y_train, y_cv = train_test_split(X, Y, test_size=0.10, random_state=42, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:24.398746Z","iopub.execute_input":"2021-08-23T14:15:24.399071Z","iopub.status.idle":"2021-08-23T14:15:25.596621Z","shell.execute_reply.started":"2021-08-23T14:15:24.399035Z","shell.execute_reply":"2021-08-23T14:15:25.595744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature engineering are inspired by this kernel --> https://www.kaggle.com/vikasmalhotra08/eda-and-lightgbm-for-avito\n\ndef feature_engineering(dataset):\n    '''This function creates the new feature with aggregating feature with mean, median, sum, min, and max'''\n    \n    # making aggregates feature with mean, sum and max.\n    data = pd.DataFrame()\n    data['price'] = dataset['price']\n    \n    # calling the text_featuring function which extract features from the title and description.\n    data[\"title_words_length\"] = dataset[\"title\"].apply(lambda x: len(x.split()))\n    data[\"description_words_length\"] = dataset[\"description\"].apply(lambda x: len(x.split()))\n    \n    data['symbol1_count'] = dataset['description'].str.count('↓')\n    data['symbol2_count'] = dataset['description'].str.count('\\*')\n    data['symbol3_count'] = dataset['description'].str.count('✔')\n    data['symbol4_count'] = dataset['description'].str.count('❀')\n    data['symbol5_count'] = dataset['description'].str.count('➚')\n    data['symbol6_count'] = dataset['description'].str.count('ஜ')\n    data['symbol7_count'] = dataset['description'].str.count('.')\n    data['symbol8_count'] = dataset['description'].str.count('!')\n    data['symbol9_count'] = dataset['description'].str.count('\\?')\n    data['symbol10_count'] = dataset['description'].str.count('  ')\n    data['symbol11_count'] = dataset['description'].str.count('-')\n    data['symbol12_count'] = dataset['description'].str.count(',') \n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:40.986283Z","iopub.execute_input":"2021-08-23T14:15:40.986644Z","iopub.status.idle":"2021-08-23T14:15:40.996261Z","shell.execute_reply.started":"2021-08-23T14:15:40.986614Z","shell.execute_reply":"2021-08-23T14:15:40.995459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test_df","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:41.878073Z","iopub.execute_input":"2021-08-23T14:15:41.87841Z","iopub.status.idle":"2021-08-23T14:15:41.881973Z","shell.execute_reply.started":"2021-08-23T14:15:41.878363Z","shell.execute_reply":"2021-08-23T14:15:41.881185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_feature_train = feature_engineering(X_train) # calling function for train set\n# num_feature_cv = feature_engineering(X_cv)       # calling function for cv set\n# num_feature_test = feature_engineering(test)     # calling function for test set","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:15:49.802187Z","iopub.execute_input":"2021-08-23T14:15:49.80258Z","iopub.status.idle":"2021-08-23T14:16:54.805418Z","shell.execute_reply.started":"2021-08-23T14:15:49.802545Z","shell.execute_reply":"2021-08-23T14:16:54.804539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let standardize our each of the numerical feature\ncolumns = num_feature_train.columns.to_list()\nfor col in columns:\n    scaler = MinMaxScaler()\n    num_feature_train[col] = scaler.fit_transform(num_feature_train[col].values.reshape(-1,1))\n    pickle.dump(scaler, open(col+'.pkl', 'wb'))\n    num_feature_cv[col] = scaler.transform(num_feature_cv[col].values.reshape(-1,1))\n    num_feature_test[col] = scaler.transform(num_feature_test[col].values.reshape(-1,1))","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:16:54.806954Z","iopub.execute_input":"2021-08-23T14:16:54.807318Z","iopub.status.idle":"2021-08-23T14:16:54.950568Z","shell.execute_reply.started":"2021-08-23T14:16:54.807283Z","shell.execute_reply":"2021-08-23T14:16:54.949434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def categorical_encoder(Train, CV, Test):\n    '''This function encode the categorical feature which we will use in NN along with embedding layer'''\n    tokeniser = Tokenizer(filters='', lower=False, split='뷁', oov_token='oov' )\n    tokeniser.fit_on_texts(Train)\n    \n    Train = np.array(tokeniser.texts_to_sequences(Train)).astype(np.float64)\n    CV = np.array(tokeniser.texts_to_sequences(CV)).astype(np.float64)\n    Test = np.array(tokeniser.texts_to_sequences(Test)).astype(np.float64)\n    length = len(tokeniser.word_index)+1\n    \n    return Train, CV, Test, length","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_type_train, user_type_cv, user_type_test, user_type_length = categorical_encoder(X_train['user_type'], X_cv['user_type'], test['user_type'])\n\nparent_category_train, parent_category_cv, parent_category_test, parent_category_length = categorical_encoder(X_train['parent_category_name'], X_cv['parent_category_name'], test['parent_category_name'])\n\ncategory_train, category_cv, category_test, category_length = categorical_encoder(X_train['category_name'], X_cv['category_name'], test['category_name'])\n\nregion_train, region_cv, region_test, region_length = categorical_encoder(X_train['region'],X_cv['region'], test['region'])\n\ncity_train, city_cv, city_test, city_length = categorical_encoder(X_train['city'], X_cv['city'], test['city'])\n\nparam_1_train, param_1_cv, param_1_test, param_1_length = categorical_encoder(X_train['param_1'], X_cv['param_1'], test['param_1'])\n\nparam_2_train, param_2_cv, param_2_test, param_2_length = categorical_encoder(X_train['param_2'], X_cv['param_2'], test['param_2'])\n\nparam_3_train,param_3_cv, param_3_test, param_3_length = categorical_encoder(X_train['param_3'], X_cv['param_3'], test['param_3'])\n\nimage_top_train, image_top_cv, image_top_test, image_top_length = categorical_encoder(X_train['image_top_1'].astype(str), X_cv['image_top_1'].astype(str), test['image_top_1'].astype(str))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# downloading the fasttext russian word vectors\n!pip install wget\nimport wget\n!wget https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ru.vec","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting the embedding index from the  russian word vector\ndef get_coefs(word, *arr): \n    return word, np.asarray(arr, dtype='float32')\n\nembeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in tqdm(open('./wiki.ru.vec', 'r', encoding='utf-8', newline='\\n', errors='ignore')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_clean(text):\n    '''This function clean the russian text'''\n    text = str(text)\n    text = text.lower()\n    clean = re.sub(r\"[,.;@#?!&$]+\\ *\", \" \", text)\n    return clean","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_features_title = 150000 # this is the maximum number of feature due to memory constraint\nmax_length_title = 8        # this is the maximum length of feature due to memory constraint\nembed_size_title = 300      # this is the maximum dimension of feature due to memory constraint\n\ntrain_df['title'] = train_df['title'].apply(text_clean)\n\ntitle_train = train_df['title'].values\n\ntokenizer_title = Tokenizer(num_words=max_features_title)\ntokenizer_title.fit_on_texts(title_train)\n\n# word_index = tokenizer_title.word_index\n# nb_words = min(max_features_title, len(word_index))\n# embedding_matrix_title = np.zeros((max_features_title, 300))\n\n# for word, i in tqdm(word_index.items()):      #generating the embedding matrix for title\n#     if i <= max_features_title:\n#         embedding_vector = embeddings_index.get(word)\n#         if embedding_vector is not None: \n#             embedding_matrix_title[i] = embedding_vector\n        \n\n# pickle.dump(embedding_matrix_title, open(\"embedding_matrix_title.pkl\", \"wb\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_features_desc = 300000           # this is the maximum number of feature due to memory constraint\nmax_length_desc = 200                # this is the maximum length of feature due to memory constraint\nembed_size_desc = 300                # this is the maximum dimension of feature due to memory constraint\n\ntrain_df['description'] = train_df['description'].apply(text_clean)\n\n\n\ndescription_train = train_df['description'].values\n\ntokenizer_desc = Tokenizer(num_words=max_features_desc)\ntokenizer_desc.fit_on_texts(description_train)\n\n# word_index = tokenizer_desc.word_index\n# nb_words = min(max_features_desc, len(word_index))\n# embedding_matrix_desc = np.zeros((nb_words, embed_size_desc))\n\n# for word, i in tqdm(word_index.items()):       #generating the embedding matrix for description\n#     if i <= max_features_desc:\n#         embedding_vector = embeddings_index.get(word)\n#         if embedding_vector is not None: \n#             embedding_matrix_desc[i] = embedding_vector\n            \n# pickle.dump(embedding_matrix_desc, open(\"embedding_matrix_desc.pkl\", \"wb\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_cv.to_csv('./y_cv.csv', index=False)\ny_train.to_csv(\"./y_train.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encoder(train, cv, test, tokenizer, max_length):\n\n  ''' This function perform the tokenization and then convert words to integers and then perform padding and returns the values '''\n    # integer encode\n  encoded_train = tokenizer.texts_to_sequences(train)\n  encoded_cv = tokenizer.texts_to_sequences(cv)\n  encoded_test = tokenizer.texts_to_sequences(test)\n\n  padded_train = np.array(pad_sequences(encoded_train, maxlen=max_length, padding='post')).astype(np.float64)\n  padded_cv = np.array(pad_sequences(encoded_cv, maxlen=max_length, padding='post')).astype(np.float64)\n  padded_test = np.array(pad_sequences(encoded_test, maxlen=max_length, padding='post')).astype(np.float64)\n\n\n  return padded_train, padded_cv, padded_test\n\npadded_title_train, padded_title_cv, padded_title_test = encoder(X_train['title'], X_cv['title'], test['title'], tokenizer_title, max_length_title)\n\npadded_desc_train, padded_desc_cv, padded_desc_test = encoder(X_train['description'], X_cv['description'], test['description'], tokenizer_desc, max_length_desc)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ndel train_df\ndel test_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting all the inputs into a list\ntrain_final = [padded_title_train, padded_desc_train, user_type_train,parent_category_train,category_train,region_train,city_train,\\\n               param_1_train, param_2_train, param_3_train,image_top_train, num_feature_train.to_numpy().astype(np.float64)]\n\ncv_final = [padded_title_cv, padded_desc_cv, user_type_cv, parent_category_cv,category_cv,region_cv,city_cv,\n               param_1_cv, param_2_cv, param_3_cv, image_top_cv, num_feature_cv.to_numpy().astype(np.float64)]\n\ntest_final = [padded_title_test, padded_desc_test, user_type_test,parent_category_test,category_test,region_test,city_test,\n               param_1_test, param_2_test, param_3_test, image_top_test, num_feature_test.to_numpy().astype(np.float64)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_feature_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(train_final, open(\"train_final.pkl\", \"wb\"))\npickle.dump(cv_final, open(\"cv_final.pkl\", \"wb\"))\npickle.dump(test_final, open(\"test_final.pkl\", \"wb\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_matrix_title = pickle.load(open('./embedding_matrix_title.pkl', 'rb'))\nembedding_matrix_desc = pickle.load(open('./embedding_matrix_desc.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:58:59.879598Z","iopub.execute_input":"2021-08-23T10:58:59.879966Z","iopub.status.idle":"2021-08-23T10:59:04.867654Z","shell.execute_reply.started":"2021-08-23T10:58:59.87992Z","shell.execute_reply":"2021-08-23T10:59:04.866366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_matrix_title = embedding_matrix_title[0:,0:100]\nembedding_matrix_desc = embedding_matrix_desc[0:,0:100]","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:04.869591Z","iopub.execute_input":"2021-08-23T10:59:04.870067Z","iopub.status.idle":"2021-08-23T10:59:04.876749Z","shell.execute_reply.started":"2021-08-23T10:59:04.87Z","shell.execute_reply":"2021-08-23T10:59:04.875402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final = pickle.load(open('train_final.pkl', 'rb'))\ncv_final = pickle.load(open('cv_final.pkl', 'rb'))\n#test_final = pickle.load(open('test_final.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:04.879298Z","iopub.execute_input":"2021-08-23T10:59:04.880126Z","iopub.status.idle":"2021-08-23T10:59:09.560153Z","shell.execute_reply.started":"2021-08-23T10:59:04.880065Z","shell.execute_reply":"2021-08-23T10:59:09.558947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final = pickle.load(open('test_final.pkl', 'rb'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = pd.read_csv('y_train.csv')\ny_cv = pd.read_csv('y_cv.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:09.562455Z","iopub.execute_input":"2021-08-23T10:59:09.562973Z","iopub.status.idle":"2021-08-23T10:59:09.767965Z","shell.execute_reply.started":"2021-08-23T10:59:09.562918Z","shell.execute_reply":"2021-08-23T10:59:09.766816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_features_desc = 300000\nmax_length_desc = 200\nembed_size_desc = 100\nmax_features_title = 150000\nmax_length_title = 8\nembed_size_title = 100\n\n\nuser_type_length = 5\nparent_category_length = 11\ncategory_length = 49 \nregion_length = 30\ncity_length = 1728\nparam_1_length = 372\nparam_2_length = 270\nparam_3_length = 1202\nimage_top_length = 3064","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:09.769637Z","iopub.execute_input":"2021-08-23T10:59:09.770111Z","iopub.status.idle":"2021-08-23T10:59:09.776552Z","shell.execute_reply.started":"2021-08-23T10:59:09.770054Z","shell.execute_reply":"2021-08-23T10:59:09.775312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras.backend as K\ndef RMSE(y_true, y_pred):\n        return K.sqrt(K.mean(K.square(y_pred - y_true)))","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:09.779401Z","iopub.execute_input":"2021-08-23T10:59:09.780153Z","iopub.status.idle":"2021-08-23T10:59:09.846965Z","shell.execute_reply.started":"2021-08-23T10:59:09.780105Z","shell.execute_reply":"2021-08-23T10:59:09.84578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dense_layer(X, units, drop):\n    X = Dense(units, activation='relu', kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001))(X)\n    X = BatchNormalization()(X)\n    X = Dropout(drop)(X)\n    return X","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:09.848917Z","iopub.execute_input":"2021-08-23T10:59:09.849397Z","iopub.status.idle":"2021-08-23T10:59:09.858194Z","shell.execute_reply.started":"2021-08-23T10:59:09.849351Z","shell.execute_reply":"2021-08-23T10:59:09.854986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_NN(length_title, length_desc, vocab_size_title, vocab_size_desc, embedded_matrix_title, embedded_matrix_desc):\n    \n  embedded_title = Embedding(vocab_size_title, 100, weights=[embedded_matrix_title], trainable= False)\n  input_0 = Input(shape= (length_title,), name='title')\n  emd_title = embedded_title(input_0)                  #Input of the title feature with embedding matrix\n\n  lstm_0 = GRU(64, activation='relu', return_sequences=True, recurrent_dropout=0.3, kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001))(emd_title)\n  flatten_0 = Flatten()(lstm_0)    #LSTM layer for title feature\n  \n  embedded_desc = Embedding(vocab_size_desc, 100, weights=[embedded_matrix_desc], trainable= False)\n  input_1 = Input(shape= (length_desc,), name='description')\n  emd_desc = embedded_desc(input_1)                   #Inputs of the description feature with embedding matrix\n\n  lstm_1 = GRU(128, activation='relu', return_sequences=True, recurrent_dropout=0.3, kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001))(emd_desc)\n  flatten_1 = Flatten()(lstm_1)      #LSTM layer for decription feature\n  \n  input_2 = Input(shape=(1,), name='user_type')             \n  emd_user_type = Embedding(user_type_length, 10) (input_2)  #Embedding layer for User_Type categorical feature\n  flatten_2 = Flatten()(emd_user_type)\n  \n  input_3 = Input(shape=(1,), name='parent_category_name')\n  emd_parent_cat = Embedding(parent_category_length, 10) (input_3)  #Embedding layer for Parent_Category_name categorical feature\n  flatten_3 = Flatten()(emd_parent_cat)\n  \n  input_4 = Input(shape=(1,), name='category_name')\n  emd_category = Embedding(category_length, 10) (input_4)           #Embedding layer for Category_name categorical feature\n  flatten_4 = Flatten()(emd_category)\n  \n  input_5 = Input(shape=(1,), name='region')\n  emd_region = Embedding(region_length, 10) (input_5)              #Embedding layer for region categorical feature\n  flatten_5 = Flatten()(emd_region)\n  \n  input_6 = Input(shape=(1,), name='city')\n  emd_city = Embedding(city_length, 10) (input_6)                  #Embedding layer for city categorical feature\n  flatten_6 = Flatten()(emd_city)\n\n  input_7 = Input(shape=(1,), name='param_1')\n  emd_param_1 = Embedding(param_1_length, 10) (input_7)            #Embedding layer for param_1 categorical feature\n  flatten_7 = Flatten()(emd_param_1)\n\n  input_8 = Input(shape=(1,), name='param_2')\n  emd_param_2 = Embedding(param_2_length, 10) (input_8)           #Embedding layer for param_2 categorical feature\n  flatten_8 = Flatten()(emd_param_2)\n  \n  input_9 = Input(shape=(1,), name='param_3')\n  emd_param_3 = Embedding(param_3_length, 10) (input_9)            #Embedding layer for param_3 categorical feature\n  flatten_9 = Flatten()(emd_param_3)\n  \n  input_10 = Input(shape=(1,), name='image_top_1')\n  emd_image_top = Embedding(image_top_length, 10) (input_10)        #Embedding layer for image_top_1 categorical feature\n  flatten_10 = Flatten()(emd_image_top)\n  \n  input_11 = Input(shape=(15,), name='Engg_features')               #inputs of the numerical feature engg\n  dense_0 = Dense(32, activation='relu', kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001)) (input_11)\n    \n  concat = concatenate([flatten_0,flatten_1,flatten_2,flatten_3,flatten_4,flatten_5,flatten_6,flatten_7,flatten_8,flatten_9,flatten_10,dense_0], name='concat_categorcal_vars')\n  #concatenating all the layers\n\n  dense_1 = Dense(512, activation='relu', kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001)) (concat)\n  BN_0 = BatchNormalization() (dense_1)\n  \n  dense_2 = Dense(256, activation='relu', kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001)) (BN_0)\n  BN_1 = BatchNormalization() (dense_2)\n\n  dense_3 = Dense(128, activation='relu', kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001)) (BN_1)\n  dropout_0 = Dropout(0.6) (dense_3)\n\n  dense_4 = Dense(64, activation='relu', kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001)) (dropout_0)\n  dropout_1 = Dropout(0.6) (dense_4)\n\n  output_layer = Dense(1, activation='linear', kernel_initializer=tf.keras.initializers.HeNormal(), kernel_regularizer=l2(0.001)) (dropout_1)\n  \n  model = Model(inputs = [input_0,input_1,input_2,input_3,input_4,input_5,input_6,input_7,input_8,input_9,input_10,input_11], outputs=output_layer)\n  optim = Adam(learning_rate=0.001)\n    \n  model.compile(loss=RMSE, optimizer=optim, metrics= [tf.keras.metrics.RootMeanSquaredError()])\n\n  tf.keras.utils.plot_model(model, show_shapes=True, to_file='base_model_multichannel.png')\n\n  return model","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:09.86037Z","iopub.execute_input":"2021-08-23T10:59:09.861132Z","iopub.status.idle":"2021-08-23T10:59:09.890207Z","shell.execute_reply.started":"2021-08-23T10:59:09.861067Z","shell.execute_reply":"2021-08-23T10:59:09.88914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\nfrom livelossplot.inputs.keras import PlotLossesCallback\n\nplot_loss = PlotLossesCallback()\n\n# ModelCheckpoint callback - save best weights\ncheckpoint = ModelCheckpoint(filepath='baseline_model_v1.weights.best.hdf5',\n                                  save_best_only=True,\n                                  verbose=1)\n\n# EarlyStopping\nearly_stop = EarlyStopping(monitor='val_loss', min_delta=0.00005,\n                           patience=5, \n                           restore_best_weights=True,\n                           mode='auto')\n#reduce Learning Rate\nreduceLR = ReduceLROnPlateau(monitor='val_root_mean_squared_error', min_lr=0.0000001,patience=2, factor=0.5, mode='auto')\n\n#terminate on NaN\nterminate = TerminateOnNaN()\n\ntensorboard = tf.keras.callbacks.TensorBoard(log_dir = \"logs/fit\" , histogram_freq = 1)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:09.891815Z","iopub.execute_input":"2021-08-23T10:59:09.892397Z","iopub.status.idle":"2021-08-23T10:59:10.162798Z","shell.execute_reply.started":"2021-08-23T10:59:09.892338Z","shell.execute_reply":"2021-08-23T10:59:10.161511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model = create_NN(max_length_title, max_length_desc, max_features_title, max_features_desc, embedding_matrix_title, embedding_matrix_desc)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:10.165134Z","iopub.execute_input":"2021-08-23T10:59:10.165599Z","iopub.status.idle":"2021-08-23T10:59:15.865197Z","shell.execute_reply.started":"2021-08-23T10:59:10.165553Z","shell.execute_reply":"2021-08-23T10:59:15.863567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\ntf.keras.backend.clear_session()\nhistory = base_model.fit(train_final, y_train, validation_data=(cv_final, y_cv), epochs=30, verbose=2, batch_size=4096, callbacks = [checkpoint, early_stop, plot_loss, tensorboard, reduceLR, terminate])","metadata":{"execution":{"iopub.status.busy":"2021-08-23T10:59:15.868309Z","iopub.execute_input":"2021-08-23T10:59:15.869144Z","iopub.status.idle":"2021-08-23T12:54:34.187056Z","shell.execute_reply.started":"2021-08-23T10:59:15.869087Z","shell.execute_reply":"2021-08-23T12:54:34.185869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = base_model.predict(test_final)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['predict'] = predict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {'item_id': test_df['item_id'], 'deal_probability': test_df['predict']}\n\nsubmission_baseline = pd.DataFrame(data, columns=['item_id', 'deal_probability'])\n\nsubmission_baseline.to_csv('./GRU_v2.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}