{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport math\nimport numpy as np \nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import r2_score\nfrom sklearn.ensemble import RandomForestRegressor\n\nfrom matplotlib import pyplot as plt\nimport seaborn as sns","metadata":{"papermill":{"duration":1.345196,"end_time":"2022-11-29T08:26:24.993490","exception":false,"start_time":"2022-11-29T08:26:23.648294","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:04.097220Z","iopub.execute_input":"2022-12-22T13:38:04.097621Z","iopub.status.idle":"2022-12-22T13:38:04.927354Z","shell.execute_reply.started":"2022-12-22T13:38:04.097545Z","shell.execute_reply":"2022-12-22T13:38:04.926601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1) Розуміння задачі\nЗавантажуємо файли з кореневої папки, test та train файли","metadata":{"papermill":{"duration":0.004364,"end_time":"2022-11-29T08:26:25.002870","exception":false,"start_time":"2022-11-29T08:26:24.998506","status":"completed"},"tags":[]}},{"cell_type":"code","source":"INPUT_PATH = \"/kaggle/input/predict-volcanic-eruptions-ingv-oe\"\nINPUT_PATH_TRAIN = INPUT_PATH + \"/train/\"\nINPUT_PATH_TEST = INPUT_PATH + \"/test/\"\n\ndf_train = pd.read_csv(INPUT_PATH + \"/train.csv\")\ndf_samplesub = pd.read_csv(INPUT_PATH + \"/sample_submission.csv\")\n\nprint(\"train.csv:\\n\", df_train)\nprint(\"\\nsample_submission.csv:\\n\", df_samplesub)","metadata":{"papermill":{"duration":0.049683,"end_time":"2022-11-29T08:26:25.056984","exception":false,"start_time":"2022-11-29T08:26:25.007301","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:04.928699Z","iopub.execute_input":"2022-12-22T13:38:04.928948Z","iopub.status.idle":"2022-12-22T13:38:04.969833Z","shell.execute_reply.started":"2022-12-22T13:38:04.928924Z","shell.execute_reply":"2022-12-22T13:38:04.968778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Файл train.csv містить ключовий атрибут до train файлів, тому кількість файлів та рядків у train.csv співпадає\n\n","metadata":{}},{"cell_type":"code","source":"train_filenames, test_filenames = [], []\n\nfor seg_id in df_train[\"segment_id\"]:\n    train_filenames.append(INPUT_PATH_TRAIN + str(seg_id) + \".csv\")\n\nfor seg_id in df_samplesub[\"segment_id\"]:\n    test_filenames.append(INPUT_PATH_TEST + str(seg_id) + \".csv\")\n    \nprint(\"Train:\", len(train_filenames), \"files\")\nprint(\"Test:\", len(test_filenames), \"files\")","metadata":{"papermill":{"duration":0.024696,"end_time":"2022-11-29T08:26:25.087015","exception":false,"start_time":"2022-11-29T08:26:25.062319","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:04.970894Z","iopub.execute_input":"2022-12-22T13:38:04.971531Z","iopub.status.idle":"2022-12-22T13:38:04.986270Z","shell.execute_reply.started":"2022-12-22T13:38:04.971500Z","shell.execute_reply":"2022-12-22T13:38:04.985175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2) Дослідження даних","metadata":{"papermill":{"duration":0.004681,"end_time":"2022-11-29T08:26:25.096725","exception":false,"start_time":"2022-11-29T08:26:25.092044","status":"completed"},"tags":[]}},{"cell_type":"code","source":"pd.read_csv(train_filenames[0])","metadata":{"papermill":{"duration":0.138043,"end_time":"2022-11-29T08:26:25.239516","exception":false,"start_time":"2022-11-29T08:26:25.101473","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:04.988772Z","iopub.execute_input":"2022-12-22T13:38:04.989031Z","iopub.status.idle":"2022-12-22T13:38:05.139421Z","shell.execute_reply.started":"2022-12-22T13:38:04.989006Z","shell.execute_reply":"2022-12-22T13:38:05.138449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(test_filenames[0])","metadata":{"papermill":{"duration":0.141315,"end_time":"2022-11-29T08:26:25.386270","exception":false,"start_time":"2022-11-29T08:26:25.244955","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:05.140891Z","iopub.execute_input":"2022-12-22T13:38:05.141194Z","iopub.status.idle":"2022-12-22T13:38:05.273240Z","shell.execute_reply.started":"2022-12-22T13:38:05.141168Z","shell.execute_reply":"2022-12-22T13:38:05.272322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_ordered = df_train.sort_values(\"time_to_eruption\")\ndf_train_ordered","metadata":{"papermill":{"duration":0.020885,"end_time":"2022-11-29T08:26:25.412781","exception":false,"start_time":"2022-11-29T08:26:25.391896","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:05.274992Z","iopub.execute_input":"2022-12-22T13:38:05.275730Z","iopub.status.idle":"2022-12-22T13:38:05.292792Z","shell.execute_reply.started":"2022-12-22T13:38:05.275691Z","shell.execute_reply":"2022-12-22T13:38:05.291469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segment_ids_ordered = list(df_train_ordered[\"segment_id\"])\nmin_tte_id, max_tte_id = segment_ids_ordered[0], segment_ids_ordered[-1]\nprint(min_tte_id)\nprint(max_tte_id)","metadata":{"papermill":{"duration":0.017224,"end_time":"2022-11-29T08:26:25.435947","exception":false,"start_time":"2022-11-29T08:26:25.418723","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:05.294715Z","iopub.execute_input":"2022-12-22T13:38:05.295108Z","iopub.status.idle":"2022-12-22T13:38:05.302624Z","shell.execute_reply.started":"2022-12-22T13:38:05.295071Z","shell.execute_reply":"2022-12-22T13:38:05.301445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Візуалізуємо паказники датчиків одного з файлів","metadata":{}},{"cell_type":"code","source":"df_min_tte = pd.read_csv(INPUT_PATH_TRAIN + str(min_tte_id) + \".csv\")\ndf_max_tte = pd.read_csv(INPUT_PATH_TRAIN + str(max_tte_id) + \".csv\")\n\npalette = sns.color_palette(\"bright\", n_colors=11)\n\nfig = plt.figure(figsize=(18, 30))\n\nfor df, df_name, col_i in [\n    (df_min_tte, \"Min Time To Eruption\", -1), \n    (df_max_tte, \"Max Time to Eruption\", 0)\n]:\n    for sensor_i in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:\n        ax = plt.subplot(10, 2, sensor_i*2 + col_i)\n        if sensor_i == 1:\n            ax.set_title(df_name)\n        df[f\"sensor_{sensor_i}\"].plot(ax=ax, color=palette[sensor_i])\n        ax.legend(loc=\"upper right\")\n\nplt.show()","metadata":{"papermill":{"duration":3.053471,"end_time":"2022-11-29T08:26:28.495784","exception":false,"start_time":"2022-11-29T08:26:25.442313","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:05.304003Z","iopub.execute_input":"2022-12-22T13:38:05.304376Z","iopub.status.idle":"2022-12-22T13:38:08.738473Z","shell.execute_reply.started":"2022-12-22T13:38:05.304349Z","shell.execute_reply":"2022-12-22T13:38:08.737568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Гістограмма розподілення часу","metadata":{}},{"cell_type":"code","source":"ax = plt.subplot(1, 1, 1)\nsns.histplot(df_train[\"time_to_eruption\"])\nax.set_title(\"Distribution of records in timeline\")\nplt.show()","metadata":{"papermill":{"duration":0.194545,"end_time":"2022-11-29T08:26:28.702644","exception":false,"start_time":"2022-11-29T08:26:28.508099","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:08.740052Z","iopub.execute_input":"2022-12-22T13:38:08.740873Z","iopub.status.idle":"2022-12-22T13:38:08.968541Z","shell.execute_reply.started":"2022-12-22T13:38:08.740817Z","shell.execute_reply":"2022-12-22T13:38:08.967609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Гістограмма кількості пустих значень у сенсорах","metadata":{}},{"cell_type":"code","source":"train_check_dfs = [ pd.read_csv(filename, nrows=2) for filename in train_filenames ]\n\ntrain_nancounts = { column: 0 for column in train_check_dfs[0] }\n\nfor df in train_check_dfs:\n    for column in df:\n        if np.isnan(df[column][0]): \n            train_nancounts[column] += 1\n\nprint(f\"{len(train_check_dfs)} files loaded\")\n            \nfig = plt.figure(figsize=(10, 7))\nax = plt.subplot(1, 1, 1)\nsns.barplot(x=list(train_nancounts.keys()), y=list(train_nancounts.values()))\nax.set_title(\"Amount of NaN from each sensor [ train ]\")\nplt.show()\n\n_ = gc.collect()","metadata":{"papermill":{"duration":60.35572,"end_time":"2022-11-29T08:27:29.068015","exception":false,"start_time":"2022-11-29T08:26:28.712295","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:38:08.971368Z","iopub.execute_input":"2022-12-22T13:38:08.971667Z","iopub.status.idle":"2022-12-22T13:39:17.429339Z","shell.execute_reply.started":"2022-12-22T13:38:08.971640Z","shell.execute_reply":"2022-12-22T13:39:17.428617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_check_dfs = [ pd.read_csv(filename, nrows=2) for filename in test_filenames ]\n\ntest_nancounts = { column: 0 for column in test_check_dfs[0] }\n\nfor df in test_check_dfs:\n    for column in df:\n        if np.isnan(df[column][0]): \n            test_nancounts[column] += 1\n\nprint(f\"{len(test_check_dfs)} files loaded\")\n            \nfig = plt.figure(figsize=(10, 7))\nax = plt.subplot(1, 1, 1)\nsns.barplot(x=list(test_nancounts.keys()), y=list(test_nancounts.values()))\nax.set_title(\"Amount of NaN from each sensor [ test ]\")\nplt.show()\n\n_ = gc.collect()","metadata":{"papermill":{"duration":76.840556,"end_time":"2022-11-29T08:28:45.918640","exception":false,"start_time":"2022-11-29T08:27:29.078084","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:39:17.430606Z","iopub.execute_input":"2022-12-22T13:39:17.431223Z","iopub.status.idle":"2022-12-22T13:40:28.752629Z","shell.execute_reply.started":"2022-12-22T13:39:17.431194Z","shell.execute_reply":"2022-12-22T13:40:28.751798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3) Перетворення даних для навчання","metadata":{"papermill":{"duration":0.009836,"end_time":"2022-11-29T08:28:45.938356","exception":false,"start_time":"2022-11-29T08:28:45.928520","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Обчислення статистичних характеристик","metadata":{}},{"cell_type":"code","source":"extractors = [\n    lambda x: x.min(),\n    lambda x: x.max(),\n    lambda x: x.mean(),\n    lambda x: x.std(),\n    lambda x: x.var(),\n    lambda x: x.skew(),\n    lambda x: np.quantile(x, 0.01),\n    lambda x: np.quantile(x, 0.05),\n    lambda x: np.quantile(x, 0.10),\n    lambda x: np.quantile(x, 0.30),\n    lambda x: np.quantile(x, 0.70),\n    lambda x: np.quantile(x, 0.90),\n    lambda x: np.quantile(x, 0.95),\n    lambda x: np.quantile(x, 0.99)\n]\n\ndef convert_one_signal(signal):\n    if not signal.count():\n        return [0]*len(extractors)\n    return [ f(signal) for f in extractors ]\n\ndef convert_one_record(record):\n    result = []\n    for sensor_id in record: \n         for t in convert_one_signal(record[sensor_id]):\n            result.append(t)\n    return result\n\ndef convert_files(file_list):\n    converted = [ convert_one_record(pd.read_csv(filename)) for filename in file_list ]\n    _ = gc.collect()\n    return np.nan_to_num(np.array(converted))","metadata":{"papermill":{"duration":0.022635,"end_time":"2022-11-29T08:28:45.970996","exception":false,"start_time":"2022-11-29T08:28:45.948361","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:40:28.753759Z","iopub.execute_input":"2022-12-22T13:40:28.754044Z","iopub.status.idle":"2022-12-22T13:40:28.763628Z","shell.execute_reply.started":"2022-12-22T13:40:28.754000Z","shell.execute_reply":"2022-12-22T13:40:28.762865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_converted = convert_files(train_filenames)\n\nprint(train_converted.shape)","metadata":{"papermill":{"duration":665.827307,"end_time":"2022-11-29T08:39:51.808295","exception":false,"start_time":"2022-11-29T08:28:45.980988","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-22T13:40:28.764779Z","iopub.execute_input":"2022-12-22T13:40:28.765312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_converted\nY = np.array(df_train[\"time_to_eruption\"])\n\nx_train, x_eval, y_train, y_eval = train_test_split(X, Y, test_size = 0.3)\n\nprint(\"Training:\", x_train.shape[0], \"records\")\nprint(\"Evaluation:\", x_eval.shape[0], \"records\")","metadata":{"papermill":{"duration":0.027382,"end_time":"2022-11-29T08:39:51.847897","exception":false,"start_time":"2022-11-29T08:39:51.820515","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4) Моделювання","metadata":{"papermill":{"duration":0.010051,"end_time":"2022-11-29T08:39:51.869902","exception":false,"start_time":"2022-11-29T08:39:51.859851","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Будуємо модель регресії методом випадкових лісів","metadata":{}},{"cell_type":"code","source":"model = RandomForestRegressor(max_depth=19)\nmodel.fit(x_train, y_train)","metadata":{"papermill":{"duration":24.904052,"end_time":"2022-11-29T08:40:16.784239","exception":false,"start_time":"2022-11-29T08:39:51.880187","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5) Оцінка точності моделі","metadata":{"papermill":{"duration":0.010032,"end_time":"2022-11-29T08:40:16.804613","exception":false,"start_time":"2022-11-29T08:40:16.794581","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Тестуємо дані для перевірки","metadata":{}},{"cell_type":"markdown","source":"Візуалізація відповідності дійсних та обчислених значень","metadata":{}},{"cell_type":"code","source":"y_pred = model.predict(x_eval)\nprint(\"- M. sq. error :\", mean_squared_error(y_eval, y_pred))\nprint(\"- Std. dev.    :\", math.sqrt(mean_squared_error(y_eval, y_pred)))\nprint(\"- R^2 score    :\", r2_score(y_eval, y_pred), \"\\n\")\n\nfig = plt.figure(figsize=(9, 9))\nax = plt.subplot(1, 1, 1)\nplt.scatter(y_eval, y_pred, color=\"orange\")\nax.set_xlabel(\"Real Time To Eruption\")\nax.set_ylabel(\"Predicted Time To Eruption\")\nplt.show()","metadata":{"papermill":{"duration":0.22368,"end_time":"2022-11-29T08:40:17.038821","exception":false,"start_time":"2022-11-29T08:40:16.815141","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 6) Впровадження моделі","metadata":{"papermill":{"duration":0.010118,"end_time":"2022-11-29T08:40:17.059333","exception":false,"start_time":"2022-11-29T08:40:17.049215","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Обчислення результату за test даними","metadata":{}},{"cell_type":"code","source":"X_test_converted = convert_files(test_filenames)\ndf_samplesub[\"time_to_eruption\"] = model.predict(X_test_converted)\ndf_samplesub","metadata":{"papermill":{"duration":697.954578,"end_time":"2022-11-29T08:51:55.024053","exception":false,"start_time":"2022-11-29T08:40:17.069475","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Створення вихідного файлу","metadata":{}},{"cell_type":"code","source":"df_samplesub.to_csv(\"submission.csv\", header=True, index=False)\nprint(\"done\")","metadata":{"papermill":{"duration":0.036895,"end_time":"2022-11-29T08:51:55.073488","exception":false,"start_time":"2022-11-29T08:51:55.036593","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}