{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\n\nwith zipfile.ZipFile('../input/avito-demand-prediction/train_jpg_0.zip', 'r') as zip_file:\n    zip_file.extractall('train_jpg_0')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_0 = pd.read_csv(\"/kaggle/input/df-train-0/df_train_0.csv\")\ndf_train_0.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_0.image = '/kaggle/working/train_jpg_0/' + df_train_0.image\ndf_train_0.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_0 = df_train_0.drop(['user_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'user_type', 'title', 'description'], axis = 1)\ndf_train_0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_0=df_train_0[['image', 'price', 'item_seq_number', 'image_top_1', 'deal_probability', 'month', 'weekday']]\ndf_train_0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train_0=df_train_0[['image', 'price', 'item_seq_number', 'image_top_1', 'deal_probability', 'month', 'weekday', 'user_id_le', 'region_le', 'city_le', 'parent_category_name_mean_price', 'category_name_mean_price', 'param_1_le', 'param_2_le', 'param_3_le']]\n# df_train_0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\ncontent = os.listdir('train_jpg_0')\nlen(content)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.remove(\"/kaggle/working/train_jpg_0/4f029e2a00e892aa2cac27d98b52ef8b13d91471f613c8d3c38e3f29d4da0b0c.jpg\")\nos.remove(\"/kaggle/working/train_jpg_0/8513a91e55670c709069b5f85e12a59095b802877715903abef16b7a6f306e58.jpg\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"content = os.listdir('train_jpg_0')\nlen(content)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install \"numpy>=1.16.5,<1.23.0\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\ndatagen = ImageDataGenerator(rescale = 1. / 255.)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Generator_for_images(dataframe, datagen, target_size, batch_size):\n generator = datagen.flow_from_dataframe(dataframe = dataframe,\n                                         x_col = 'image',\n                                         y_col = 'deal_probability',\n                                         batch_size = batch_size,\n                                         #class_mode = \"other\",\n                                         class_mode = \"raw\",\n                                         color_mode = 'rgb',\n                                         target_size = target_size\n                                         )\n return generator\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# генератор возвращающий [изображения, доп. данные], целевой_вектор\n# необходим для построения модели с несколькими входными данными\ndef Generator_for_image_and_numeric_data(generator, batch_size, df):\n count = 0\n while True:\n     if count == len(df.index):\n         generator.reset()\n         break\n     count += batch_size\n     batch = next(generator)\n        \n     img = batch[0]\n     extra_data = batch[1][:,:3]\n     targets = batch[1][:,3:]\n     yield [img, extra_data], targets","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size_train = 32\nbatch_size_valid = 32\nbatch_size_test = 2000","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set, test_valid_set = train_test_split(df_train_0, train_size = 0.8, random_state = 17)\ntest_set, valid_set = train_test_split(test_valid_set, train_size = 0.5, random_state = 17)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_train_generator = Generator_for_images(train_set, datagen, (224, 224), batch_size_train)\nimage_valid_generator = Generator_for_images(valid_set, datagen, (224, 224), batch_size_valid)\nimage_test_generator = Generator_for_images(test_set, datagen, (224, 224),batch_size_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = Generator_for_image_and_numeric_data(image_train_generator,\nbatch_size_train, train_set)\nvalid_generator = Generator_for_image_and_numeric_data(image_valid_generator,\nbatch_size_valid, valid_set)\ntest_generator = Generator_for_image_and_numeric_data(image_test_generator,\nbatch_size_test, test_set)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# построение модели на основе MobileNet\nfrom keras.applications.mobilenet import MobileNet\nfrom keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Add, Activation, BatchNormalization\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.models import Sequential, Model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# модель на основе MobileNet с несколькими входными данными\nbase_mobilenet_model = MobileNet(input_shape = (224,224,3), include_top = False)\nfor layer in base_mobilenet_model.layers:\n layer.trainable = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_image = Sequential()\nmodel_image.add(base_mobilenet_model)\nmodel_image.add(GlobalAveragePooling2D())\nmodel_image.add(Dropout(0.5))\nmodel_image.add(Dense(256, activation = 'relu'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_extra_data = Sequential()\nmodel_extra_data.add(Dense(512,input_shape = (3,), activation = 'relu'))\nmodel_extra_data.add(Dropout(0.2))\nmodel_extra_data.add(Dense(256, activation='relu'))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_merged = Add()([model_image.output, model_extra_data.output])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_merged = Flatten()(model_merged)\nmodel_merged = Dense(256, activation='relu')(model_merged)\nmodel_merged = Dropout(0.5)(model_merged)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# выходной слой\n#model_merged = Dense(len(disease_labels), activation='sigmoid')(model_merged)\nmodel_merged = Dense(1, activation='relu')(model_merged)\nmodel_3 = Model([model_image.input,model_extra_data.input], model_merged)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow_addons.metrics import RSquare\nmodel_3.compile(optimizer = 'adam', loss = 'mse', metrics = [RSquare()])\nmodel_3.summary()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# графическое представление модели\nfrom keras.utils.vis_utils import plot_model\nplot_model(model_3, show_shapes=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator_steps = train_set.shape[0] // batch_size_train\nvalid_generator_steps = valid_set.shape[0] // batch_size_valid","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_3.fit_generator(generator = train_generator,\n    steps_per_epoch = train_generator_steps,\n    epochs = 50,\n    validation_data = valid_generator,\n    validation_steps = valid_generator_steps)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}