{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-25T18:50:15.147791Z","iopub.execute_input":"2023-05-25T18:50:15.148215Z","iopub.status.idle":"2023-05-25T18:50:15.180346Z","shell.execute_reply.started":"2023-05-25T18:50:15.148182Z","shell.execute_reply":"2023-05-25T18:50:15.179476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-05-25T07:03:05.280892Z","iopub.execute_input":"2023-05-25T07:03:05.281329Z","iopub.status.idle":"2023-05-25T07:03:05.303829Z","shell.execute_reply.started":"2023-05-25T07:03:05.281298Z","shell.execute_reply":"2023-05-25T07:03:05.302573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:09:34.986976Z","iopub.execute_input":"2023-05-25T19:09:34.987395Z","iopub.status.idle":"2023-05-25T19:09:34.992566Z","shell.execute_reply.started":"2023-05-25T19:09:34.987363Z","shell.execute_reply":"2023-05-25T19:09:34.991371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\n\nwith zipfile.ZipFile('../input/avito-demand-prediction/train_jpg_0.zip', 'r') as zip_file:\n    zip_file.extractall('train_jpg_0')","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:09:36.532079Z","iopub.execute_input":"2023-05-25T19:09:36.53248Z","iopub.status.idle":"2023-05-25T19:12:34.343032Z","shell.execute_reply.started":"2023-05-25T19:09:36.532447Z","shell.execute_reply":"2023-05-25T19:12:34.341414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_0 = pd.read_csv(\"/kaggle/input/df-train-0/df_train_0.csv\")\ndf_train_0.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:13:01.175825Z","iopub.execute_input":"2023-05-25T19:13:01.176307Z","iopub.status.idle":"2023-05-25T19:13:09.065736Z","shell.execute_reply.started":"2023-05-25T19:13:01.176261Z","shell.execute_reply":"2023-05-25T19:13:09.064662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_0.image = '/kaggle/working/train_jpg_0/' + df_train_0.image\ndf_train_0 = df_train_0[[\"image\", \"deal_probability\"]]\ndf_train_0.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:13:14.799926Z","iopub.execute_input":"2023-05-25T19:13:14.800452Z","iopub.status.idle":"2023-05-25T19:13:15.133824Z","shell.execute_reply.started":"2023-05-25T19:13:14.800408Z","shell.execute_reply":"2023-05-25T19:13:15.132565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_result = pd.read_csv(\"/kaggle/input/cat-result/cat_result.csv\")\ncat_result.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:13:28.116509Z","iopub.execute_input":"2023-05-25T19:13:28.116939Z","iopub.status.idle":"2023-05-25T19:13:28.402909Z","shell.execute_reply.started":"2023-05-25T19:13:28.116908Z","shell.execute_reply":"2023-05-25T19:13:28.401926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in cat_result.index:\n    cat_result.prediction[i] = cat_result.prediction[i].replace(']', '')\n    cat_result.prediction[i] = cat_result.prediction[i].replace('[', '')\ncat_result.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:13:32.932309Z","iopub.execute_input":"2023-05-25T19:13:32.932721Z","iopub.status.idle":"2023-05-25T19:14:26.86782Z","shell.execute_reply.started":"2023-05-25T19:13:32.932689Z","shell.execute_reply":"2023-05-25T19:14:26.866739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_result = cat_result.astype({'prediction': float})\ncat_result.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:14:26.870102Z","iopub.execute_input":"2023-05-25T19:14:26.870521Z","iopub.status.idle":"2023-05-25T19:14:26.937137Z","shell.execute_reply.started":"2023-05-25T19:14:26.870482Z","shell.execute_reply":"2023-05-25T19:14:26.936062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_0['cat_result']= cat_result['prediction']\ndf_train_0.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:14:31.976343Z","iopub.execute_input":"2023-05-25T19:14:31.976758Z","iopub.status.idle":"2023-05-25T19:14:31.990085Z","shell.execute_reply.started":"2023-05-25T19:14:31.976725Z","shell.execute_reply":"2023-05-25T19:14:31.98897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"content = os.listdir('train_jpg_0')\nlen(content)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:14:56.882151Z","iopub.execute_input":"2023-05-25T19:14:56.882577Z","iopub.status.idle":"2023-05-25T19:14:57.149448Z","shell.execute_reply.started":"2023-05-25T19:14:56.882528Z","shell.execute_reply":"2023-05-25T19:14:57.148275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.remove(\"/kaggle/working/train_jpg_0/4f029e2a00e892aa2cac27d98b52ef8b13d91471f613c8d3c38e3f29d4da0b0c.jpg\")\nos.remove(\"/kaggle/working/train_jpg_0/8513a91e55670c709069b5f85e12a59095b802877715903abef16b7a6f306e58.jpg\")","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:15:00.260788Z","iopub.execute_input":"2023-05-25T19:15:00.261164Z","iopub.status.idle":"2023-05-25T19:15:00.267041Z","shell.execute_reply.started":"2023-05-25T19:15:00.261135Z","shell.execute_reply":"2023-05-25T19:15:00.265638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"content = os.listdir('train_jpg_0')\nlen(content)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:15:03.245957Z","iopub.execute_input":"2023-05-25T19:15:03.246354Z","iopub.status.idle":"2023-05-25T19:15:03.521575Z","shell.execute_reply.started":"2023-05-25T19:15:03.246323Z","shell.execute_reply":"2023-05-25T19:15:03.520517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install \"numpy>=1.16.5,<1.23.0\"","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:15:06.166817Z","iopub.execute_input":"2023-05-25T19:15:06.167251Z","iopub.status.idle":"2023-05-25T19:15:26.75572Z","shell.execute_reply.started":"2023-05-25T19:15:06.167216Z","shell.execute_reply":"2023-05-25T19:15:26.754336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(rescale = 1. / 255.)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:16:23.515828Z","iopub.execute_input":"2023-05-25T19:16:23.516668Z","iopub.status.idle":"2023-05-25T19:16:23.522585Z","shell.execute_reply.started":"2023-05-25T19:16:23.516632Z","shell.execute_reply":"2023-05-25T19:16:23.521333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:16:25.778636Z","iopub.execute_input":"2023-05-25T19:16:25.779035Z","iopub.status.idle":"2023-05-25T19:16:26.524471Z","shell.execute_reply.started":"2023-05-25T19:16:25.779006Z","shell.execute_reply":"2023-05-25T19:16:26.523525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set, test_valid_set = train_test_split(df_train_0, train_size = 0.8, random_state = 17)\ntest_set, valid_set = train_test_split(test_valid_set, train_size = 0.5, random_state = 17)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:16:31.301832Z","iopub.execute_input":"2023-05-25T19:16:31.302333Z","iopub.status.idle":"2023-05-25T19:16:31.393294Z","shell.execute_reply.started":"2023-05-25T19:16:31.302291Z","shell.execute_reply":"2023-05-25T19:16:31.392347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Generator_for_images(dataframe, datagen, target_size, batch_size):\n generator = datagen.flow_from_dataframe(dataframe = dataframe,\n                                         x_col = 'image',\n                                         y_col = 'deal_probability',\n                                         batch_size = batch_size,\n                                         class_mode = \"raw\",\n                                         color_mode = 'rgb',\n                                         target_size = target_size,\n                                         shuffle=False\n                                         )\n return generator","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:16:52.441677Z","iopub.execute_input":"2023-05-25T19:16:52.442069Z","iopub.status.idle":"2023-05-25T19:16:52.449243Z","shell.execute_reply.started":"2023-05-25T19:16:52.442039Z","shell.execute_reply":"2023-05-25T19:16:52.448102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# генератор возвращающий [изображения, доп. данные], целевой_вектор\n# необходим для построения модели с несколькими входными данными\ndef Generator_for_image_and_numeric_data(generator, batch_size, df):\n count = 0\n while True:\n     if count == len(df.index):\n         generator.reset()\n         break\n     count += batch_size\n     batch = next(generator)\n        \n     img = batch[0]\n     extra_data = batch[1][:,:3]\n     targets = batch[1][:,3:]\n     yield [img, extra_data], targets","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:17:54.162565Z","iopub.execute_input":"2023-05-25T19:17:54.162972Z","iopub.status.idle":"2023-05-25T19:17:54.171883Z","shell.execute_reply.started":"2023-05-25T19:17:54.162942Z","shell.execute_reply":"2023-05-25T19:17:54.170264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size_train = 32\nbatch_size_valid = 32\nbatch_size_test = 2000","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:18:19.799362Z","iopub.execute_input":"2023-05-25T19:18:19.799759Z","iopub.status.idle":"2023-05-25T19:18:19.804767Z","shell.execute_reply.started":"2023-05-25T19:18:19.79973Z","shell.execute_reply":"2023-05-25T19:18:19.803702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_train_generator = Generator_for_images(train_set, datagen, (224, 224), batch_size_train)\nimage_valid_generator = Generator_for_images(valid_set, datagen, (224, 224), batch_size_valid)\nimage_test_generator = Generator_for_images(test_set, datagen, (224, 224), batch_size_test)     ","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:18:41.7514Z","iopub.execute_input":"2023-05-25T19:18:41.752691Z","iopub.status.idle":"2023-05-25T19:18:45.354751Z","shell.execute_reply.started":"2023-05-25T19:18:41.752646Z","shell.execute_reply":"2023-05-25T19:18:45.353508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = Generator_for_image_and_numeric_data(image_train_generator,\nbatch_size_train, train_set)\nvalid_generator = Generator_for_image_and_numeric_data(image_valid_generator,\nbatch_size_valid, valid_set)\ntest_generator = Generator_for_image_and_numeric_data(image_test_generator,\nbatch_size_test, test_set)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def Hamming_Loss(y_true, y_pred):\n#  #losses = tf.TensorArray(tf.float32, size=len(y_true))\n#  losses = []\n#  for i in range(len(y_true)):\n#      count = 0\n#      for j in range(len(y_true[0])):\n#          if (y_pred[i][j] != y_true[i][j]):\n#              count += 1\n#      #losses = losses.write(i, count/len(y_true[i]))\n#      losses.append(count/len(y_true[i]))\n#  #return tf.convert_to_tensor(losses)\n#  return losses","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:21:00.247601Z","iopub.execute_input":"2023-05-25T19:21:00.25122Z","iopub.status.idle":"2023-05-25T19:21:00.276469Z","shell.execute_reply.started":"2023-05-25T19:21:00.251049Z","shell.execute_reply":"2023-05-25T19:21:00.273782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def Accuracy(y_true, y_pred):\n#  #accuracy = tf.TensorArray(tf.float32, size=len(y_true))\n#  accuracy = []\n#  for i in range(len(y_true)):\n#      count1, count2 = 0, 0\n#      for j in range(len(y_true[0])):\n#          count1 += (y_pred[i][j] and y_true[i][j])\n#          count2 += (y_pred[i][j] or y_true[i][j])\n#      #accuracy = accuracy.write(i, count1 / count2)\n#      accuracy.append(count1 / count2)\n#  #return tf.convert_to_tensor(accuracy)\n#  return accuracy","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:21:52.503241Z","iopub.execute_input":"2023-05-25T19:21:52.503703Z","iopub.status.idle":"2023-05-25T19:21:52.511092Z","shell.execute_reply.started":"2023-05-25T19:21:52.503666Z","shell.execute_reply":"2023-05-25T19:21:52.509965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# построение модели на основе MobileNet\nfrom keras.applications.mobilenet import MobileNet\nfrom keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Add, Activation, BatchNormalization\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.models import Sequential, Model","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:22:22.944454Z","iopub.execute_input":"2023-05-25T19:22:22.944915Z","iopub.status.idle":"2023-05-25T19:22:22.951723Z","shell.execute_reply.started":"2023-05-25T19:22:22.944878Z","shell.execute_reply":"2023-05-25T19:22:22.950453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# модель на основе MobileNet с несколькими входными данными\nbase_mobilenet_model = MobileNet(input_shape = (224,224,3), include_top = False)\nfor layer in base_mobilenet_model.layers:\n layer.trainable = False","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:22:32.528041Z","iopub.execute_input":"2023-05-25T19:22:32.528444Z","iopub.status.idle":"2023-05-25T19:22:35.741113Z","shell.execute_reply.started":"2023-05-25T19:22:32.528416Z","shell.execute_reply":"2023-05-25T19:22:35.739841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_image = Sequential()\nmodel_image.add(base_mobilenet_model)\nmodel_image.add(GlobalAveragePooling2D())\nmodel_image.add(Dropout(0.5))\nmodel_image.add(Dense(256, activation = 'relu'))","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:22:42.585214Z","iopub.execute_input":"2023-05-25T19:22:42.585666Z","iopub.status.idle":"2023-05-25T19:22:42.893787Z","shell.execute_reply.started":"2023-05-25T19:22:42.585632Z","shell.execute_reply":"2023-05-25T19:22:42.892768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_extra_data = Sequential()\nmodel_extra_data.add(Dense(512,input_shape = (3,), activation = 'relu'))\nmodel_extra_data.add(Dropout(0.2))\nmodel_extra_data.add(Dense(256, activation='relu'))\n","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:23:00.440815Z","iopub.execute_input":"2023-05-25T19:23:00.441205Z","iopub.status.idle":"2023-05-25T19:23:00.487409Z","shell.execute_reply.started":"2023-05-25T19:23:00.441175Z","shell.execute_reply":"2023-05-25T19:23:00.486341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_merged = Add()([model_image.output, model_extra_data.output])","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:23:08.504284Z","iopub.execute_input":"2023-05-25T19:23:08.50468Z","iopub.status.idle":"2023-05-25T19:23:08.516631Z","shell.execute_reply.started":"2023-05-25T19:23:08.504651Z","shell.execute_reply":"2023-05-25T19:23:08.515071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_merged = Flatten()(model_merged)\nmodel_merged = Dense(256, activation='relu')(model_merged)\nmodel_merged = Dropout(0.5)(model_merged)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:23:12.719528Z","iopub.execute_input":"2023-05-25T19:23:12.719972Z","iopub.status.idle":"2023-05-25T19:23:12.749366Z","shell.execute_reply.started":"2023-05-25T19:23:12.719937Z","shell.execute_reply":"2023-05-25T19:23:12.74835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# выходной слой\n#model_merged = Dense(len(disease_labels), activation='sigmoid')(model_merged)\nmodel_merged = Dense(1, activation='relu')(model_merged)\nmodel_3 = Model([model_image.input,model_extra_data.input], model_merged)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:24:03.891821Z","iopub.execute_input":"2023-05-25T19:24:03.892249Z","iopub.status.idle":"2023-05-25T19:24:03.922032Z","shell.execute_reply.started":"2023-05-25T19:24:03.892219Z","shell.execute_reply":"2023-05-25T19:24:03.920926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow_addons.metrics import RSquare\nmodel_3.compile(optimizer = 'adam', loss = 'mse', metrics = [RSquare()])\n# model_3.compile(optimizer = 'adam', loss = Hamming_Loss, metrics = Accuracy)\nmodel_3.summary()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:25:55.058207Z","iopub.execute_input":"2023-05-25T19:25:55.058709Z","iopub.status.idle":"2023-05-25T19:25:55.862337Z","shell.execute_reply.started":"2023-05-25T19:25:55.058674Z","shell.execute_reply":"2023-05-25T19:25:55.861311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# графическое представление модели\nfrom keras.utils.vis_utils import plot_model\nplot_model(model_3, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:26:08.555914Z","iopub.execute_input":"2023-05-25T19:26:08.556374Z","iopub.status.idle":"2023-05-25T19:26:08.885Z","shell.execute_reply.started":"2023-05-25T19:26:08.556331Z","shell.execute_reply":"2023-05-25T19:26:08.883527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator_steps = train_set.shape[0] \nvalid_generator_steps = test_valid_set.shape[0] ","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:26:19.792731Z","iopub.execute_input":"2023-05-25T19:26:19.793123Z","iopub.status.idle":"2023-05-25T19:26:19.798194Z","shell.execute_reply.started":"2023-05-25T19:26:19.793093Z","shell.execute_reply":"2023-05-25T19:26:19.796978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_3.fit_generator(generator = train_generator,\n                      steps_per_epoch = train_generator_steps,\n                      epochs = 1,\n                      validation_data = valid_generator,\n                      validation_steps = valid_generator_steps)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T19:27:24.055114Z","iopub.execute_input":"2023-05-25T19:27:24.055843Z","iopub.status.idle":"2023-05-25T19:27:25.349462Z","shell.execute_reply.started":"2023-05-25T19:27:24.055803Z","shell.execute_reply":"2023-05-25T19:27:25.347813Z"},"trusted":true},"execution_count":null,"outputs":[]}]}