{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/predict-volcanic-eruptions-ingv-oe/train.csv\")\nsample_submission = pd.read_csv(\"../input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.express as px\n\nfig = px.histogram(\n    train, \n    x=\"time_to_eruption\",\n    width=800,\n    height=500,\n    nbins=100,\n    title='Время до извержения'\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.line(\n    train, \n    y=\"time_to_eruption\",\n    width=800,\n    height=500,\n    title='Время до извержения всех вулканов'\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['time_to_eruption'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Median:', train['time_to_eruption'].median())\nprint('Skew:', train['time_to_eruption'].skew())\nprint('Std:', train['time_to_eruption'].std())\nprint('Kurtosis:', train['time_to_eruption'].kurtosis())\nprint('Mean:', train['time_to_eruption'].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/train/2037160701.csv')\ncheck","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\n\ntrain_frags = glob.glob(\"../input/predict-volcanic-eruptions-ingv-oe/train/*\")\nlen(train_frags)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sensors = set()\nobservations = set()\nnan_columns = list()\nmissed_groups = list()\nfor_df = list()\n\nfor item in train_frags:\n    name = int(item.split('.')[-2].split('/')[-1])\n    at_least_one_missed = 0\n    frag = pd.read_csv(item)\n    missed_group = list()\n    missed_percents = list()\n    for col in frag.columns:\n        missed_percents.append(frag[col].isnull().sum() / len(frag))\n        if pd.isnull(frag[col]).all() == True:\n            at_least_one_missed = 1\n            nan_columns.append(col)\n            missed_group.append(col)\n    if len(missed_group) > 0:\n        missed_groups.append(missed_group)\n    sensors.add(len(frag.columns))\n    observations.add(len(frag))\n    for_df.append([name, at_least_one_missed] + missed_percents)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Количество уникальных сенсоров: ', sensors)\nprint('Количество уникальных наблюдений: ', observations)\nprint('Количество пропущенных сенсоров:', len(nan_columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"absent_sensors = dict()\n\nfor item in nan_columns:\n    if item in absent_sensors:\n        absent_sensors[item] += 1\n    else:\n        absent_sensors[item] = 0\n\nabsent_df = pd.DataFrame(absent_sensors.items(), columns=['Сенсор', 'Сколько раз пропущен'])\n\nfig = px.bar(\n    absent_df, \n    x='Сколько раз пропущен',\n    y='Сенсор',\n    width=800,\n    height=600,\n    title='Количество пропущенных сенсоров в train датасете'\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"absent_groups = dict()\n\nfor item in missed_groups:\n    if str(item) in absent_groups:\n        absent_groups[str(item)] += 1\n    else:\n        absent_groups[str(item)] = 0\n\nabsent_df = pd.DataFrame(absent_groups.items(), columns=['Группы', 'Пропущена'])\nabsent_df = absent_df.sort_values('Пропущена')\n\nfig = px.bar(\n    absent_df, \n    y=\"Группы\",\n    x='Пропущена',\n    orientation='h',\n    width=800,\n    height=600,\n    title='Количество пропущенных групп сенсоров в train датасете'\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for_df = pd.DataFrame(\n    for_df, \n    columns=[\n        'segment_id', 'has_missed_sensors', 'missed_percent_sensor1', \n        'missed_percent_sensor2', 'missed_percent_sensor3', 'missed_percent_sensor4', \n        'missed_percent_sensor5', 'missed_percent_sensor6', 'missed_percent_sensor7', \n        'missed_percent_sensor8', 'missed_percent_sensor9', 'missed_percent_sensor10'\n    ]\n)\n\nfor_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.merge(train, for_df)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from plotly.subplots import make_subplots\nimport plotly.graph_objs as go\n\nfig = make_subplots(rows=1, cols=2)\ntraces = [\n    go.Histogram(\n        x=train[train['has_missed_sensors']==1]['time_to_eruption'], \n        nbinsx=200, \n        name='Есть пропущенные сенсоры'\n    ),\n    go.Histogram(\n        x=train[train['has_missed_sensors']==0]['time_to_eruption'], \n        nbinsx=200, \n        name=\"Нет пропущенных сенсоров\"\n    )\n]\n\nfor i in range(len(traces)):\n    fig.append_trace(\n        traces[i], \n        (i // 2) + 1, \n        (i % 2) + 1\n    )\n\nfig.update_layout(\n    title_text='Время до извержения для частей с пропущенными сенсорами и без',\n    height=600,\n    width=1200\n)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_frags = glob.glob(\"../input/predict-volcanic-eruptions-ingv-oe/test/*\")\nlen(test_frags)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sensors = set()\nobservations = set()\nnan_columns = list()\nmissed_groups = list()\nfor_test_df = list()\n\nfor item in test_frags:\n    name = int(item.split('.')[-2].split('/')[-1])\n    at_least_one_missed = 0\n    frag = pd.read_csv(item)\n    missed_group = list()\n    missed_percents = list()\n    for col in frag.columns:\n        missed_percents.append(frag[col].isnull().sum() / len(frag))\n        if pd.isnull(frag[col]).all() == True:\n            at_least_one_missed = 1\n            nan_columns.append(col)\n            missed_group.append(col)\n    if len(missed_group) > 0:\n        missed_groups.append(missed_group)\n    sensors.add(len(frag.columns))\n    observations.add(len(frag))\n    for_test_df.append([name, at_least_one_missed] + missed_percents)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for_test_df = pd.DataFrame(\n    for_test_df, \n    columns=[\n        'segment_id', 'has_missed_sensors', 'missed_percent_sensor1', 'missed_percent_sensor2', 'missed_percent_sensor3', \n        'missed_percent_sensor4', 'missed_percent_sensor5', 'missed_percent_sensor6', 'missed_percent_sensor7', \n        'missed_percent_sensor8', 'missed_percent_sensor9', 'missed_percent_sensor10'\n    ]\n)\n\nfor_test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Количество уникальных сенсоров: ', sensors)\nprint('Количество уникальных наблюдений: ', observations)\nprint('Количество пропущенных сенсоров:', len(nan_columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"absent_sensors = dict()\n\nfor item in nan_columns:\n    if item in absent_sensors:\n        absent_sensors[item] += 1\n    else:\n        absent_sensors[item] = 0\n\nabsent_df = pd.DataFrame(absent_sensors.items(), columns=['Сенсор', 'Сколько раз пропущен'])\n\nfig = px.bar(\n    absent_df, \n    x='Сколько раз пропущен',\n    y='Сенсор',\n    width=800,\n    height=600,\n    title='Количество пропущенных сенсоров в test датасете'\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"absent_groups = dict()\n\nfor item in missed_groups:\n    if str(item) in absent_groups:\n        absent_groups[str(item)] += 1\n    else:\n        absent_groups[str(item)] = 0\n\nabsent_df = pd.DataFrame(absent_groups.items(), columns=['Группы', 'Сколько раз пропущена'])\nabsent_df = absent_df.sort_values('Сколько раз пропущена')\n\nfig = px.bar(\n    absent_df, \n    y='Группы',\n    x='Сколько раз пропущена',\n    orientation='h',\n    width=800,\n    height=600,\n    title='Количество пропущенных групп сенсоров в test датасете'\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = make_subplots(rows=5, cols=2)\ntraces = [\n    go.Histogram(\n        x=check[col], \n        nbinsx=100, \n        name=col\n    ) for col in check.columns\n]\n\nfor i in range(len(traces)):\n    fig.append_trace(\n        traces[i], \n        (i // 2) + 1, \n        (i % 2) + 1\n    )\n\nfig.update_layout(\n    title_text='Данные о распределении сенсоров',\n    height=800,\n    width=1200\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = make_subplots(rows=5, cols=2)\ntraces = [\n    go.Scatter(\n        x=[i for i in range(60002)], \n        y=check[col], \n        mode='lines', \n        name=col\n    ) for col in check.columns\n]\n\nfor i in range(len(traces)):\n    fig.append_trace(\n        traces[i], \n        (i // 2) + 1, \n        (i % 2) + 1\n    )\n\nfig.update_layout(\n    title_text='Данные о сенсорах',\n    height=800,\n    width=1200\n)\n\nfig.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}