{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nplt.style.use('seaborn-white')\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nimport lightgbm as lgbm\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom tqdm import tqdm\n\nfile_list = []\nfile_list_train = []\nfile_list_test = []\n\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        file_list.append(os.path.join(dirname, filename))\n        \nPATH = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'\n\nfor dirname, _, filenames in os.walk('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train'):\n    for filename in filenames:\n        file_list_train.append(os.path.join(dirname, filename))\n        \nfor dirname, _, filenames in os.walk('/kaggle/input/predict-volcanic-eruptions-ingv-oe/test'):\n    for filename in filenames:\n        file_list_test.append(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-15T10:05:53.237632Z","iopub.execute_input":"2022-11-15T10:05:53.238082Z","iopub.status.idle":"2022-11-15T10:05:58.168901Z","shell.execute_reply.started":"2022-11-15T10:05:53.238047Z","shell.execute_reply":"2022-11-15T10:05:58.167543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Segment_id - номер сегменту з якого знімаються покази датчиків.\nTime_to_eruption - час до виверження**","metadata":{}},{"cell_type":"code","source":"print(file_list[0])\n\n\nprint(pd.read_csv(file_list[0]))\nprint(pd.read_csv(file_list[0]).isna().sum())","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:05:58.171282Z","iopub.execute_input":"2022-11-15T10:05:58.171788Z","iopub.status.idle":"2022-11-15T10:05:58.199729Z","shell.execute_reply.started":"2022-11-15T10:05:58.171741Z","shell.execute_reply":"2022-11-15T10:05:58.198287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Папка train містить файли з показниками показів датчиків***","metadata":{}},{"cell_type":"markdown","source":"***Десять хвилин журналів від десяти різних датчиків,\nрозташованих навколо вулкана***","metadata":{}},{"cell_type":"code","source":"print(file_list_train[0])\nprint(pd.read_csv(file_list_train[0]))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:05:58.201445Z","iopub.execute_input":"2022-11-15T10:05:58.201925Z","iopub.status.idle":"2022-11-15T10:05:58.31915Z","shell.execute_reply.started":"2022-11-15T10:05:58.201879Z","shell.execute_reply":"2022-11-15T10:05:58.317886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(file_list[1])\nprint(pd.read_csv(file_list[1]))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:05:58.322125Z","iopub.execute_input":"2022-11-15T10:05:58.323487Z","iopub.status.idle":"2022-11-15T10:05:58.342451Z","shell.execute_reply.started":"2022-11-15T10:05:58.323433Z","shell.execute_reply":"2022-11-15T10:05:58.341111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Папка test містить набори значень датчиків але вже без відповідного значення\nчасу до виверження***","metadata":{}},{"cell_type":"code","source":"print(file_list_test[0])\nprint(pd.read_csv(file_list_test[0]))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:05:58.344751Z","iopub.execute_input":"2022-11-15T10:05:58.345631Z","iopub.status.idle":"2022-11-15T10:05:58.423062Z","shell.execute_reply.started":"2022-11-15T10:05:58.34558Z","shell.execute_reply":"2022-11-15T10:05:58.421616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint(len(file_list_test)) \nkeys = list(pd.read_csv(file_list_test[0]).keys()) \nprint(keys)\nnanC = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\nfor index in range(len(file_list_test)):\n    if(index % 200 == 0): \n        print(index)\n    df = pd.read_csv(file_list_test[index]) \n    for key in df.keys():\n        if df[key].isna().sum() == len(pd.read_csv(file_list_train[0])):            \n            nanC[keys.index(key)] += 1\n\nprint(nanC)\ndata={'sensors': keys, 'count': nanC}\nNaNDF = pd.DataFrame(data)\nprint(NaNDF)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:05:58.425075Z","iopub.execute_input":"2022-11-15T10:05:58.425559Z","iopub.status.idle":"2022-11-15T10:15:42.90644Z","shell.execute_reply.started":"2022-11-15T10:05:58.425496Z","shell.execute_reply":"2022-11-15T10:15:42.905142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NaNDF.plot(figsize =(20, 20), x=\"sensors\", y=\"count\", kind=\"bar\",  rot=5, fontsize=14 )","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:42.908309Z","iopub.execute_input":"2022-11-15T10:15:42.909556Z","iopub.status.idle":"2022-11-15T10:15:43.315967Z","shell.execute_reply.started":"2022-11-15T10:15:42.909492Z","shell.execute_reply":"2022-11-15T10:15:43.314618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Кількість файлів у тестовому наборі ***\n","metadata":{}},{"cell_type":"code","source":"print(len(file_list_test))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.317749Z","iopub.execute_input":"2022-11-15T10:15:43.31815Z","iopub.status.idle":"2022-11-15T10:15:43.324296Z","shell.execute_reply.started":"2022-11-15T10:15:43.318115Z","shell.execute_reply":"2022-11-15T10:15:43.322785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Кількість файлів у наборі для тренування***","metadata":{}},{"cell_type":"code","source":"print(len(file_list_train))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.325851Z","iopub.execute_input":"2022-11-15T10:15:43.326217Z","iopub.status.idle":"2022-11-15T10:15:43.341268Z","shell.execute_reply.started":"2022-11-15T10:15:43.326173Z","shell.execute_reply":"2022-11-15T10:15:43.340025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Перевірка чи наявні файли з однією назвою в наборах для навчання та тренування***","metadata":{}},{"cell_type":"code","source":"files_train = [file.split('/')[-1].split('.')[-2] for file in file_list_train]\nfiles_test = [file.split('/')[-1].split('.')[-2] for file in file_list_test]","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.34619Z","iopub.execute_input":"2022-11-15T10:15:43.346673Z","iopub.status.idle":"2022-11-15T10:15:43.361202Z","shell.execute_reply.started":"2022-11-15T10:15:43.346633Z","shell.execute_reply":"2022-11-15T10:15:43.35976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Перші 10 файлів тренування***","metadata":{}},{"cell_type":"code","source":"print(files_train[0:10])","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.362795Z","iopub.execute_input":"2022-11-15T10:15:43.363162Z","iopub.status.idle":"2022-11-15T10:15:43.382237Z","shell.execute_reply.started":"2022-11-15T10:15:43.36313Z","shell.execute_reply":"2022-11-15T10:15:43.3808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(files_test[0:10])","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.384363Z","iopub.execute_input":"2022-11-15T10:15:43.385091Z","iopub.status.idle":"2022-11-15T10:15:43.396355Z","shell.execute_reply.started":"2022-11-15T10:15:43.385045Z","shell.execute_reply":"2022-11-15T10:15:43.394911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Кількість перетинів індексів файлів***","metadata":{}},{"cell_type":"code","source":"test_set = set(files_test)\ntrain_set = set(files_train)\ninter = test_set.intersection(train_set)\n\nprint(inter)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.397925Z","iopub.execute_input":"2022-11-15T10:15:43.398373Z","iopub.status.idle":"2022-11-15T10:15:43.4369Z","shell.execute_reply.started":"2022-11-15T10:15:43.398334Z","shell.execute_reply":"2022-11-15T10:15:43.435323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Для подальшої роботи зчитуємо дані тренування у Data Frame***","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(PATH+'train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.438487Z","iopub.execute_input":"2022-11-15T10:15:43.43901Z","iopub.status.idle":"2022-11-15T10:15:43.452895Z","shell.execute_reply.started":"2022-11-15T10:15:43.438961Z","shell.execute_reply":"2022-11-15T10:15:43.451415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Діаграма розподілення часу.***","metadata":{}},{"cell_type":"code","source":"sns.distplot(train['time_to_eruption'],\n            hist=True,\n            kde=True,\n            bins=100,\n            color='blue',\n            hist_kws={'edgecolor':'black'})","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.454728Z","iopub.execute_input":"2022-11-15T10:15:43.455235Z","iopub.status.idle":"2022-11-15T10:15:43.901856Z","shell.execute_reply.started":"2022-11-15T10:15:43.455182Z","shell.execute_reply":"2022-11-15T10:15:43.900623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Описова статистика ознаки часу***","metadata":{}},{"cell_type":"code","source":"train['time_to_eruption'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.904186Z","iopub.execute_input":"2022-11-15T10:15:43.905028Z","iopub.status.idle":"2022-11-15T10:15:43.919416Z","shell.execute_reply.started":"2022-11-15T10:15:43.90497Z","shell.execute_reply":"2022-11-15T10:15:43.917755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Візуалізуємо покази датчиків. Наприклад, вже розглянутого раніше файлу train/800654756.csv***","metadata":{}},{"cell_type":"code","source":"df_segment_id = pd.read_csv(PATH+'train/800654756.csv')\n\ndf_segment_id.plot(figsize=(20,20),\n                  subplots=True,\n                  layout=(10,1),\n                  rot=0,\n                  lw=1,\n                  title='sergment_id #800654756')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:43.921514Z","iopub.execute_input":"2022-11-15T10:15:43.922005Z","iopub.status.idle":"2022-11-15T10:15:46.516025Z","shell.execute_reply.started":"2022-11-15T10:15:43.921965Z","shell.execute_reply":"2022-11-15T10:15:46.514798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Візуалізуємо\nпокази, які відповідають мінімальному та максимальному часу.***","metadata":{}},{"cell_type":"code","source":"display(train.sort_values('time_to_eruption', axis=0, ascending=True).iloc[[0,-1],:])\n\nsegment_id_min = 601524801\nsegment_id_max = 1923243961\n\ndf_segment_id_min = pd.read_csv(PATH+'train/'+str(segment_id_min)+'.csv')\ndf_segment_id_max = pd.read_csv(PATH+'train/'+str(segment_id_max)+'.csv')","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:46.517572Z","iopub.execute_input":"2022-11-15T10:15:46.518042Z","iopub.status.idle":"2022-11-15T10:15:46.7206Z","shell.execute_reply.started":"2022-11-15T10:15:46.518008Z","shell.execute_reply":"2022-11-15T10:15:46.719634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_segment_id_min.plot(figsize=(20,20), subplots=True, layout=(10,1), rot=0, lw=1, title='segment_id #601524801 (min)')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:46.723604Z","iopub.execute_input":"2022-11-15T10:15:46.724271Z","iopub.status.idle":"2022-11-15T10:15:49.222544Z","shell.execute_reply.started":"2022-11-15T10:15:46.724233Z","shell.execute_reply":"2022-11-15T10:15:49.221259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_segment_id_max.plot(figsize=(20,20), subplots=True, layout=(10,1), rot=0, lw=1, title='segment_id #1923243961 (max)')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:49.224437Z","iopub.execute_input":"2022-11-15T10:15:49.224871Z","iopub.status.idle":"2022-11-15T10:15:51.993319Z","shell.execute_reply.started":"2022-11-15T10:15:49.224834Z","shell.execute_reply":"2022-11-15T10:15:51.99199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Як бачимо покази, що відповідають максимальному часу відрізняються за\n> характеристиками амплітуди та частоти від показів, які відповідають мінімальному\n> часу. Конкретна інтерпретація цих показів неможливо без знань типів датчиків та не\n> входить до нашої задачі. \n","metadata":{}},{"cell_type":"markdown","source":"***Підготовка даних для навчання (Data Preparation)***","metadata":{}},{"cell_type":"code","source":"def build_features(signal, ts, sensor_id):\n    X = pd.DataFrame()\n    f = np.fft.fft(signal)\n    f_real = np.real(f)\n    X.loc[ts, f'{sensor_id}_sum'] = signal.sum()\n    X.loc[ts, f'{sensor_id}_mean'] = signal.mean()\n    X.loc[ts, f'{sensor_id}_std'] = signal.std()\n    X.loc[ts, f'{sensor_id}_var'] = signal.var()\n    X.loc[ts, f'{sensor_id}_max'] = signal.max()\n    X.loc[ts, f'{sensor_id}_min'] = signal.min()\n    X.loc[ts, f'{sensor_id}_skew'] = signal.skew()\n    X.loc[ts, f'{sensor_id}_mad'] = signal.mad()\n    X.loc[ts, f'{sensor_id}_kurtosis'] = signal.kurtosis()\n    X.loc[ts, f'{sensor_id}_quantile99'] = np.quantile(signal, 0.99)\n    X.loc[ts, f'{sensor_id}_quantile95'] = np.quantile(signal, 0.95)\n    X.loc[ts, f'{sensor_id}_quantile85'] = np.quantile(signal, 0.85)\n    X.loc[ts, f'{sensor_id}_quantile75'] = np.quantile(signal, 0.75)\n    X.loc[ts, f'{sensor_id}_quantile55'] = np.quantile(signal, 0.55)\n    X.loc[ts, f'{sensor_id}_quantile45'] = np.quantile(signal, 0.45)\n    X.loc[ts, f'{sensor_id}_quantile25'] = np.quantile(signal, 0.25)\n    X.loc[ts, f'{sensor_id}_quantile15'] = np.quantile(signal, 0.15)\n    X.loc[ts, f'{sensor_id}_quantile05'] = np.quantile(signal, 0.05)\n    X.loc[ts, f'{sensor_id}_quantile01'] = np.quantile(signal, 0.01)\n    X.loc[ts, f'{sensor_id}_fft_real_mean'] = f_real.mean()\n    X.loc[ts, f'{sensor_id}_fft_real_std'] = f_real.std()\n    X.loc[ts, f'{sensor_id}_fft_real_max'] = f_real.max()\n    X.loc[ts, f'{sensor_id}_fft_real_min'] = f_real.min()\n    \n    return X","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:51.995089Z","iopub.execute_input":"2022-11-15T10:15:51.995899Z","iopub.status.idle":"2022-11-15T10:15:52.015628Z","shell.execute_reply.started":"2022-11-15T10:15:51.995826Z","shell.execute_reply":"2022-11-15T10:15:52.014313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Формування DataFrame з даними тренування***","metadata":{}},{"cell_type":"code","source":"train_set = list()\nseg = 0\n\nfor seg, segment_id in enumerate(train.segment_id):\n    signals = pd.read_csv(PATH+'train/'+str(segment_id)+'.csv')\n    train_row = []\n    \n    if seg%200 == 0:\n        print('Processing segment_id={}'.format(seg))\n        \n    for sensor in range(0, 10):\n        sensor_id = f'sensor_{sensor+1}'\n        train_row.append(build_features(signals[sensor_id].fillna(0), segment_id, sensor_id))\n        \n    train_row = pd.concat(train_row, axis=1)\n    train_set.append(train_row)\n    seg+=1\n    \ntrain_set = pd.concat(train_set)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:15:52.017244Z","iopub.execute_input":"2022-11-15T10:15:52.018497Z","iopub.status.idle":"2022-11-15T10:59:29.736428Z","shell.execute_reply.started":"2022-11-15T10:15:52.018391Z","shell.execute_reply":"2022-11-15T10:59:29.735044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.reset_index()\ntrain_set = train_set.rename(columns={'index':  'segment_id'})\n\ntrain_set = pd.merge(train_set, train, on='segment_id')","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:59:29.738503Z","iopub.execute_input":"2022-11-15T10:59:29.739347Z","iopub.status.idle":"2022-11-15T10:59:29.804131Z","shell.execute_reply.started":"2022-11-15T10:59:29.739297Z","shell.execute_reply":"2022-11-15T10:59:29.80292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_set.head(3))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:59:29.806191Z","iopub.execute_input":"2022-11-15T10:59:29.806701Z","iopub.status.idle":"2022-11-15T10:59:29.825737Z","shell.execute_reply.started":"2022-11-15T10:59:29.806658Z","shell.execute_reply":"2022-11-15T10:59:29.824001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Дані для тесту**","metadata":{}},{"cell_type":"code","source":"test_files = []\nfor dirname, _, filenames in os.walk(PATH+'test/'):\n    for filename in filenames:\n        test_files.append(filename[:-4])\n        \ntest = pd.DataFrame(test_files, columns=['segment_id'])","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:59:29.827796Z","iopub.execute_input":"2022-11-15T10:59:29.828172Z","iopub.status.idle":"2022-11-15T10:59:30.635287Z","shell.execute_reply.started":"2022-11-15T10:59:29.828139Z","shell.execute_reply":"2022-11-15T10:59:30.634032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = list()\nseg = 0\n\nfor seg, segment_id in enumerate(test.segment_id):\n    signals = pd.read_csv(PATH+'test/'+str(segment_id)+'.csv')\n    test_row = []\n    \n    if seg%200 == 0:\n        print('Processing segment_id={}'.format(seg))\n        \n    for sensor in range(0, 10):\n        sensor_id = f'sensor_{sensor+1}'\n        test_row.append(build_features(signals[sensor_id].fillna(0), segment_id, sensor_id))\n        \n    test_row = pd.concat(test_row, axis=1)\n    test_set.append(test_row)\n    seg+=1\n    \ntest_set = pd.concat(test_set)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T10:59:30.636859Z","iopub.execute_input":"2022-11-15T10:59:30.637246Z","iopub.status.idle":"2022-11-15T11:44:05.012122Z","shell.execute_reply.started":"2022-11-15T10:59:30.63721Z","shell.execute_reply":"2022-11-15T11:44:05.011065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = test_set.reset_index()\ntest_set = test_set.rename(columns={'index':  'segment_id'})\n\ntest_set = pd.merge(test_set, test, on='segment_id')","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:05.013616Z","iopub.execute_input":"2022-11-15T11:44:05.013988Z","iopub.status.idle":"2022-11-15T11:44:05.054132Z","shell.execute_reply.started":"2022-11-15T11:44:05.013955Z","shell.execute_reply":"2022-11-15T11:44:05.053032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_set.head(3))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:05.060121Z","iopub.execute_input":"2022-11-15T11:44:05.060827Z","iopub.status.idle":"2022-11-15T11:44:05.080328Z","shell.execute_reply.started":"2022-11-15T11:44:05.060788Z","shell.execute_reply":"2022-11-15T11:44:05.079056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Розділення даних тренування на набори для безпосередньо тренування та валідації**","metadata":{}},{"cell_type":"code","source":"X = train_set.drop(['segment_id', 'time_to_eruption'], axis=1)\ny = train_set['time_to_eruption']\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, \n                                                      test_size=0.2,\n                                                      random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:05.081806Z","iopub.execute_input":"2022-11-15T11:44:05.082179Z","iopub.status.idle":"2022-11-15T11:44:05.103046Z","shell.execute_reply.started":"2022-11-15T11:44:05.082144Z","shell.execute_reply":"2022-11-15T11:44:05.101696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Вхідні дані тренування (статистичні характеристики показів датчиків) ","metadata":{}},{"cell_type":"code","source":"print(X_train.head(3))\nprint('np.shape(X_train) = ', np.shape(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:05.104727Z","iopub.execute_input":"2022-11-15T11:44:05.10547Z","iopub.status.idle":"2022-11-15T11:44:05.125099Z","shell.execute_reply.started":"2022-11-15T11:44:05.105415Z","shell.execute_reply":"2022-11-15T11:44:05.123668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Вихідні дані тренування (дані часу) ","metadata":{}},{"cell_type":"code","source":"print(y_train.head(3))\nprint('np.shape(y_train) = ', np.shape(y_train))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:05.126934Z","iopub.execute_input":"2022-11-15T11:44:05.127343Z","iopub.status.idle":"2022-11-15T11:44:05.140396Z","shell.execute_reply.started":"2022-11-15T11:44:05.1273Z","shell.execute_reply":"2022-11-15T11:44:05.138886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Побудуємо просту модель регресії методом Випадкових лісів. ","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n\nmodel = RandomForestRegressor(max_depth=20, random_state=0)\nmodel.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:05.144631Z","iopub.execute_input":"2022-11-15T11:44:05.145029Z","iopub.status.idle":"2022-11-15T11:44:55.648971Z","shell.execute_reply.started":"2022-11-15T11:44:05.144995Z","shell.execute_reply":"2022-11-15T11:44:55.647601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Оцінка точності моделі (Evaluation) ","metadata":{}},{"cell_type":"code","source":"y_pred = model.predict(X_valid)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:55.650913Z","iopub.execute_input":"2022-11-15T11:44:55.652041Z","iopub.status.idle":"2022-11-15T11:44:55.704673Z","shell.execute_reply.started":"2022-11-15T11:44:55.651992Z","shell.execute_reply":"2022-11-15T11:44:55.703558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\nmse = mean_squared_error(y_valid, y_pred)\nfig = plt.figure()\nmulreg = fig.add_subplot(1, 1, 1)\nmulreg.scatter(y_valid, y_pred, color='r')\nmulreg.set_title('Nonlinear Regression')","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:55.706219Z","iopub.execute_input":"2022-11-15T11:44:55.706604Z","iopub.status.idle":"2022-11-15T11:44:55.934426Z","shell.execute_reply.started":"2022-11-15T11:44:55.706569Z","shell.execute_reply":"2022-11-15T11:44:55.933484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Впровадження моделі (Deployment)\nПрогноз на тестових даних ","metadata":{}},{"cell_type":"code","source":"prediction = model.predict(test_set.drop(columns=['segment_id']))","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:55.935843Z","iopub.execute_input":"2022-11-15T11:44:55.936885Z","iopub.status.idle":"2022-11-15T11:44:56.096838Z","shell.execute_reply.started":"2022-11-15T11:44:55.936846Z","shell.execute_reply":"2022-11-15T11:44:56.095205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()  \nsubmission['segment_id'] = test_set['segment_id']\nsubmission['time_to_eruption'] = prediction\nsubmission.to_csv('submission.csv', header=True, index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T11:44:56.098428Z","iopub.execute_input":"2022-11-15T11:44:56.098804Z","iopub.status.idle":"2022-11-15T11:44:56.124036Z","shell.execute_reply.started":"2022-11-15T11:44:56.098771Z","shell.execute_reply":"2022-11-15T11:44:56.122793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nkeys = list(pd.read_csv(file_list_test[0]).keys()) \nprint(keys)\nnanC = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\nfor index in range(len(file_list_test)):\n    df = pd.read_csv(file_list_test[index]) \n    for key in df.keys():\n        if df[key].isna().sum() == len(pd.read_csv(file_list_train[0])):      \n            nanC[keys.index(key)] += 1\n\nprint(nanC)\ndata={'sensors': keys, 'count': nanC}\nNaNDF = pd.DataFrame(data)\nprint(NaNDF)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NaNDF.plot(figsize =(20, 20), x=\"sensors\", y=\"count\", kind=\"bar\",  rot=5, fontsize=14 )","metadata":{},"execution_count":null,"outputs":[]}]}