{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#데이터확인\nimport numpy as np\nimport pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport datetime\n\n\n#데이터 시각화\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n#기타\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:28.443871Z","iopub.execute_input":"2021-09-29T22:39:28.444156Z","iopub.status.idle":"2021-09-29T22:39:28.453856Z","shell.execute_reply.started":"2021-09-29T22:39:28.444126Z","shell.execute_reply":"2021-09-29T22:39:28.453139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. 데이터 확인","metadata":{}},{"cell_type":"markdown","source":"## Train.csv 확인","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/predict-volcanic-eruptions-ingv-oe/train.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:28.455405Z","iopub.execute_input":"2021-09-29T22:39:28.456061Z","iopub.status.idle":"2021-09-29T22:39:28.473151Z","shell.execute_reply.started":"2021-09-29T22:39:28.456026Z","shell.execute_reply":"2021-09-29T22:39:28.472449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:28.474326Z","iopub.execute_input":"2021-09-29T22:39:28.4746Z","iopub.status.idle":"2021-09-29T22:39:28.485589Z","shell.execute_reply.started":"2021-09-29T22:39:28.474568Z","shell.execute_reply":"2021-09-29T22:39:28.484793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csvs = glob.glob(\"../input/predict-volcanic-eruptions-ingv-oe/train/*\")\ntest_csvs = glob.glob(\"../input/predict-volcanic-eruptions-ingv-oe/test/*\")","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:28.487935Z","iopub.execute_input":"2021-09-29T22:39:28.488232Z","iopub.status.idle":"2021-09-29T22:39:28.523237Z","shell.execute_reply.started":"2021-09-29T22:39:28.48818Z","shell.execute_reply":"2021-09-29T22:39:28.522621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train segment_id 확인","metadata":{}},{"cell_type":"code","source":"train_csvs[0]  #위에 glob으로 파일패스를 만들었기떄문에 요렇게쓰면 경로지정\nsequence = pd.read_csv(train_csvs[0]) #위의 경로에있는 파일 불러오기\nsequence","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:28.525233Z","iopub.execute_input":"2021-09-29T22:39:28.525709Z","iopub.status.idle":"2021-09-29T22:39:28.6143Z","shell.execute_reply.started":"2021-09-29T22:39:28.525674Z","shell.execute_reply":"2021-09-29T22:39:28.613491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence.describe()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:28.615737Z","iopub.execute_input":"2021-09-29T22:39:28.616028Z","iopub.status.idle":"2021-09-29T22:39:28.674249Z","shell.execute_reply.started":"2021-09-29T22:39:28.615978Z","shell.execute_reply":"2021-09-29T22:39:28.673537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sensor 값 확인하기","metadata":{}},{"cell_type":"code","source":"#plt를 사용하여 간단하게 시각화\ndef show_sensors(df):\n    f, axes = plt.subplots(10, 1)\n    f.set_size_inches((16, 10)) \n    f.tight_layout()\n    plt.subplots_adjust(bottom=-0.4)\n    \n    # Sensor #1 ~ #10\n    for i in range(1,11):\n        axes[i-1].plot(df[f'sensor_{i}'].values)\n        axes[i-1].set_title('Sensor_'+str(i))\n        axes[i-1].set_xlabel('time')\n        \nshow_sensors(sequence)","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:28.67551Z","iopub.execute_input":"2021-09-29T22:39:28.675785Z","iopub.status.idle":"2021-09-29T22:39:30.480773Z","shell.execute_reply.started":"2021-09-29T22:39:28.675738Z","shell.execute_reply":"2021-09-29T22:39:30.480082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#널값을 채워준다\nshow_sensors(sequence.fillna(0))","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:30.48176Z","iopub.execute_input":"2021-09-29T22:39:30.481992Z","iopub.status.idle":"2021-09-29T22:39:32.010998Z","shell.execute_reply.started":"2021-09-29T22:39:30.481963Z","shell.execute_reply":"2021-09-29T22:39:32.01033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. 데이터 분석 - Basic EDA","metadata":{}},{"cell_type":"markdown","source":"## Train.csv\n- `time_to_eruption`을 h:m:s(시:분:초)로 변환","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:32.012152Z","iopub.execute_input":"2021-09-29T22:39:32.012601Z","iopub.status.idle":"2021-09-29T22:39:32.022848Z","shell.execute_reply.started":"2021-09-29T22:39:32.012559Z","shell.execute_reply":"2021-09-29T22:39:32.022042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 센서 값은 10분 간 `60000`개가 측정됨\n- 10분은 600(10 * 60초)초 이므로, 센서 값은 0.01초 간격으로 샘플링됨(단위가 10e-2 초 - centisecond 임)\n- `datetime.timedelta` 적용할 때, 위 특성을 적용","metadata":{}},{"cell_type":"code","source":"#위의 0.0초단위를 시:분:초로 변환\ntrain['time_to_eruption'].apply(lambda x:datetime.timedelta(seconds = x/100))","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:32.025883Z","iopub.execute_input":"2021-09-29T22:39:32.026573Z","iopub.status.idle":"2021-09-29T22:39:32.058753Z","shell.execute_reply.started":"2021-09-29T22:39:32.026538Z","shell.execute_reply":"2021-09-29T22:39:32.057967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#밑의 시각화를 위해 새로운 컬럼 생성\ntrain['hhmmss'] = train['time_to_eruption'].apply(lambda x:datetime.timedelta(seconds = x/100))\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:32.059908Z","iopub.execute_input":"2021-09-29T22:39:32.060213Z","iopub.status.idle":"2021-09-29T22:39:32.097674Z","shell.execute_reply.started":"2021-09-29T22:39:32.06018Z","shell.execute_reply":"2021-09-29T22:39:32.096965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#평균적으로 몇일뒤에 화산이 발생하는지 확인\nplt.figure(figsize=(16,8))\nplt.hist(train['hhmmss'] / pd.Timedelta(days=1))\nplt.xlabel('Time between eruptions (days)')\nplt.ylabel('# of eruptions');","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:32.100704Z","iopub.execute_input":"2021-09-29T22:39:32.100897Z","iopub.status.idle":"2021-09-29T22:39:32.32491Z","shell.execute_reply.started":"2021-09-29T22:39:32.100876Z","shell.execute_reply":"2021-09-29T22:39:32.32423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train['hhmmss'] / pd.Timedelta(hours=1)).hist()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:32.326073Z","iopub.execute_input":"2021-09-29T22:39:32.326327Z","iopub.status.idle":"2021-09-29T22:39:32.553149Z","shell.execute_reply.started":"2021-09-29T22:39:32.326294Z","shell.execute_reply":"2021-09-29T22:39:32.552494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#px로도 시각화 가능 좀더 유동적인 것을 가능\nfig = px.histogram(\n    train, \n    x=\"time_to_eruption\",\n    width=800,\n    height=600,\n    nbins=10,\n    title='Time to eruption distribution'\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:32.554271Z","iopub.execute_input":"2021-09-29T22:39:32.554539Z","iopub.status.idle":"2021-09-29T22:39:33.524899Z","shell.execute_reply.started":"2021-09-29T22:39:32.554504Z","shell.execute_reply":"2021-09-29T22:39:33.524242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(\n    train, \n    y=\"time_to_eruption\",\n    width=800,\n    height=500,\n    title='Time to eruption for all volcanos'\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:33.52745Z","iopub.execute_input":"2021-09-29T22:39:33.52768Z","iopub.status.idle":"2021-09-29T22:39:33.627097Z","shell.execute_reply.started":"2021-09-29T22:39:33.527655Z","shell.execute_reply":"2021-09-29T22:39:33.626436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train & Test 셋 분석","metadata":{}},{"cell_type":"markdown","source":"## Train 셋","metadata":{}},{"cell_type":"code","source":"#중복이 있는지 확인,센서값이 모두 10개인지 확인 널값 채우기\nsensors = set()\nobservations = set()\nnan_columns = list()\nmissed_groups = list()\nfor_df = list()\n\nfor item in train_csvs:           #train_csvs폴더에서 파일들을 하나하나 가져옴\n    name = int(item.split('.')[-2].split('/')[-1])   #파일이름에서.전까지 즉 확장자 뺴고 이름만 가져옴\n    at_least_one_missed = 0\n    frag = pd.read_csv(item)     #판다스로 데이터를 읽는다\n    missed_group = list()\n    missed_percents = list()\n    for col in frag.columns:\n        missed_percents.append(frag[col].isnull().sum() / len(frag))\n        if pd.isnull(frag[col]).all() == True:     #각각의 컬럼에대해 널값이 있는지 확\n            at_least_one_missed = 1     #널값이 있다면 1을 저\n            nan_columns.append(col)     #어펜드로 위의 non_coloumns에 넣는다\n            missed_group.append(col)\n    if len(missed_group) > 0:\n        missed_groups.append(missed_group)\n    sensors.add(len(frag.columns))\n    observations.add(len(frag))\n    for_df.append([name, at_least_one_missed] + missed_percents)","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:39:33.628531Z","iopub.execute_input":"2021-09-29T22:39:33.628975Z","iopub.status.idle":"2021-09-29T22:46:41.825486Z","shell.execute_reply.started":"2021-09-29T22:39:33.62894Z","shell.execute_reply":"2021-09-29T22:46:41.824762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('고유한(Unique) 센서 수: ', sensors)\nprint('고유한(Unique) 행(row) 수: ', observations)","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:46:41.826982Z","iopub.execute_input":"2021-09-29T22:46:41.82727Z","iopub.status.idle":"2021-09-29T22:46:41.835871Z","shell.execute_reply.started":"2021-09-29T22:46:41.827226Z","shell.execute_reply":"2021-09-29T22:46:41.835224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#넑값 체\nprint('센서 값이 비어있는(missed) 센서 수:', len(nan_columns))\n\nabsent_sensors = dict()\n\nfor item in nan_columns:\n    if item in absent_sensors:\n        absent_sensors[item] += 1\n    else:\n        absent_sensors[item] = 0","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:46:41.83698Z","iopub.execute_input":"2021-09-29T22:46:41.837315Z","iopub.status.idle":"2021-09-29T22:46:41.851061Z","shell.execute_reply.started":"2021-09-29T22:46:41.837282Z","shell.execute_reply":"2021-09-29T22:46:41.85039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#널값의 시각화\nabsent_df = pd.DataFrame(absent_sensors.items(), columns=['Sensor', 'Missed sensors'])\n\nfig = px.bar(\n    absent_df, \n    x=\"Sensor\",\n    y='Missed sensors',\n    width=800,\n    height=500,\n    title='Number of missed sensors in training dataset'\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:46:41.852271Z","iopub.execute_input":"2021-09-29T22:46:41.852548Z","iopub.status.idle":"2021-09-29T22:46:41.922387Z","shell.execute_reply.started":"2021-09-29T22:46:41.852517Z","shell.execute_reply":"2021-09-29T22:46:41.921584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test 셋","metadata":{}},{"cell_type":"code","source":"sensors = set()\nobservations = set()\nnan_columns = list()\nmissed_groups = list()\nfor_test_df = list()\n\nfor item in test_csvs:\n    name = int(item.split('.')[-2].split('/')[-1])\n    at_least_one_missed = 0\n    frag = pd.read_csv(item)\n    missed_group = list()\n    missed_percents = list()\n    for col in frag.columns:\n        missed_percents.append(frag[col].isnull().sum() / len(frag))\n        if pd.isnull(frag[col]).all() == True:\n            at_least_one_missed = 1\n            nan_columns.append(col)\n            missed_group.append(col)\n    if len(missed_group) > 0:\n        missed_groups.append(missed_group)\n    sensors.add(len(frag.columns))\n    observations.add(len(frag))\n    for_test_df.append([name, at_least_one_missed] + missed_percents)","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:46:41.923668Z","iopub.execute_input":"2021-09-29T22:46:41.923965Z","iopub.status.idle":"2021-09-29T22:53:40.868254Z","shell.execute_reply.started":"2021-09-29T22:46:41.923933Z","shell.execute_reply":"2021-09-29T22:53:40.867529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('고유한(Unique) 센서 수: ', sensors)\nprint('고유한(Unique) 행(row) 수: ', observations)","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:53:40.869615Z","iopub.execute_input":"2021-09-29T22:53:40.869987Z","iopub.status.idle":"2021-09-29T22:53:40.877835Z","shell.execute_reply.started":"2021-09-29T22:53:40.869948Z","shell.execute_reply":"2021-09-29T22:53:40.876973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('값이 비어있는(missed) 센서 수:', len(nan_columns))\n\nabsent_sensors = dict()\n\nfor item in nan_columns:\n    if item in absent_sensors:\n        absent_sensors[item] += 1\n    else:\n        absent_sensors[item] = 0","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:53:40.879472Z","iopub.execute_input":"2021-09-29T22:53:40.879956Z","iopub.status.idle":"2021-09-29T22:53:40.888762Z","shell.execute_reply.started":"2021-09-29T22:53:40.879918Z","shell.execute_reply":"2021-09-29T22:53:40.887767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"absent_df = pd.DataFrame(absent_sensors.items(), columns=['Sensor', 'Missed sensors'])\n\nfig = px.bar(\n    absent_df, \n    x=\"Sensor\",\n    y='Missed sensors',\n    width=800,\n    height=500,\n    title='Number of missed sensors in test dataset'\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T22:53:40.890164Z","iopub.execute_input":"2021-09-29T22:53:40.890507Z","iopub.status.idle":"2021-09-29T22:53:40.975756Z","shell.execute_reply.started":"2021-09-29T22:53:40.890462Z","shell.execute_reply":"2021-09-29T22:53:40.975101Z"},"trusted":true},"execution_count":null,"outputs":[]}]}