{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport glob\nfrom tqdm import tqdm\n\nimport datetime\n\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-25T11:42:11.530849Z","iopub.execute_input":"2022-01-25T11:42:11.531055Z","iopub.status.idle":"2022-01-25T11:42:13.092015Z","shell.execute_reply.started":"2022-01-25T11:42:11.531032Z","shell.execute_reply":"2022-01-25T11:42:13.091428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. 데이터확인","metadata":{}},{"cell_type":"markdown","source":"## 파일수 확인","metadata":{}},{"cell_type":"code","source":"train_csvs = glob.glob('../input/predict-volcanic-eruptions-ingv-oe/train/*')\nlen(train_csvs)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:42:23.524989Z","iopub.execute_input":"2022-01-25T11:42:23.525262Z","iopub.status.idle":"2022-01-25T11:42:23.661296Z","shell.execute_reply.started":"2022-01-25T11:42:23.525232Z","shell.execute_reply":"2022-01-25T11:42:23.660656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csvs = glob.glob(\"../input/predict-volcanic-eruptions-ingv-oe/test/*\")\nlen(test_csvs)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:42:44.157914Z","iopub.execute_input":"2022-01-25T12:42:44.158593Z","iopub.status.idle":"2022-01-25T12:42:44.437632Z","shell.execute_reply.started":"2022-01-25T12:42:44.158563Z","shell.execute_reply":"2022-01-25T12:42:44.437209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train.csv확인","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/train.csv')\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:48:04.841646Z","iopub.execute_input":"2022-01-25T11:48:04.84217Z","iopub.status.idle":"2022-01-25T11:48:04.856801Z","shell.execute_reply.started":"2022-01-25T11:48:04.842145Z","shell.execute_reply":"2022-01-25T11:48:04.856242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sample Submission 확인","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:46:27.488462Z","iopub.execute_input":"2022-01-25T11:46:27.488672Z","iopub.status.idle":"2022-01-25T11:46:27.50634Z","shell.execute_reply.started":"2022-01-25T11:46:27.48865Z","shell.execute_reply":"2022-01-25T11:46:27.505844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train segment_id확인","metadata":{}},{"cell_type":"code","source":"train_csvs[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:48:41.779069Z","iopub.execute_input":"2022-01-25T11:48:41.779516Z","iopub.status.idle":"2022-01-25T11:48:41.784724Z","shell.execute_reply.started":"2022-01-25T11:48:41.779484Z","shell.execute_reply":"2022-01-25T11:48:41.784041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence = pd.read_csv(train_csvs[0])\nsequence","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:49:11.432807Z","iopub.execute_input":"2022-01-25T11:49:11.43303Z","iopub.status.idle":"2022-01-25T11:49:11.577856Z","shell.execute_reply.started":"2022-01-25T11:49:11.433005Z","shell.execute_reply":"2022-01-25T11:49:11.577282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:50:14.046058Z","iopub.execute_input":"2022-01-25T11:50:14.046287Z","iopub.status.idle":"2022-01-25T11:50:14.095241Z","shell.execute_reply.started":"2022-01-25T11:50:14.046264Z","shell.execute_reply":"2022-01-25T11:50:14.094833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sensor 값 확인","metadata":{}},{"cell_type":"code","source":"def show_sensors(df):\n    f,axes = plt.subplots(10,1)\n    f.set_size_inches((16,10))\n    f.tight_layout()\n    plt.subplots_adjust(bottom=-0.4)\n    \n    for i in  range (1,11):\n        axes[i-1].plot(df[f'sensor_{i}'].values)\n        axes[i-1].set_title('Sensor_'+str(i))\n        axes[i-1].set_xlabel('time')","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:54:28.231907Z","iopub.execute_input":"2022-01-25T11:54:28.232369Z","iopub.status.idle":"2022-01-25T11:54:28.238398Z","shell.execute_reply.started":"2022-01-25T11:54:28.232332Z","shell.execute_reply":"2022-01-25T11:54:28.23777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sensors(sequence)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:54:30.914214Z","iopub.execute_input":"2022-01-25T11:54:30.914519Z","iopub.status.idle":"2022-01-25T11:54:32.304824Z","shell.execute_reply.started":"2022-01-25T11:54:30.914482Z","shell.execute_reply":"2022-01-25T11:54:32.304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sensors(sequence.fillna(0))","metadata":{"execution":{"iopub.status.busy":"2022-01-25T11:56:37.383947Z","iopub.execute_input":"2022-01-25T11:56:37.384182Z","iopub.status.idle":"2022-01-25T11:56:38.699433Z","shell.execute_reply.started":"2022-01-25T11:56:37.384159Z","shell.execute_reply":"2022-01-25T11:56:38.698684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.데이터분석, EDA\n## Train.csv","metadata":{}},{"cell_type":"code","source":"train['time_to_eruption'].apply(lambda x:datetime.timedelta(seconds = x/100))\n# 시간단위 변환","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:01:37.32299Z","iopub.execute_input":"2022-01-25T12:01:37.323221Z","iopub.status.idle":"2022-01-25T12:01:37.342503Z","shell.execute_reply.started":"2022-01-25T12:01:37.323195Z","shell.execute_reply":"2022-01-25T12:01:37.341631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['hhmmss'] = train['time_to_eruption'].apply(lambda x:datetime.timedelta(seconds = x/100))\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:02:49.162213Z","iopub.execute_input":"2022-01-25T12:02:49.162434Z","iopub.status.idle":"2022-01-25T12:02:49.185956Z","shell.execute_reply.started":"2022-01-25T12:02:49.162411Z","shell.execute_reply":"2022-01-25T12:02:49.185443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train['hhmmss']/pd.Timedelta(hours=1)).hist()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:05:32.942824Z","iopub.execute_input":"2022-01-25T12:05:32.943492Z","iopub.status.idle":"2022-01-25T12:05:33.244254Z","shell.execute_reply.started":"2022-01-25T12:05:32.943462Z","shell.execute_reply":"2022-01-25T12:05:33.243642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(\n      train,\n      x='time_to_eruption',\n      width=800, height=600, nbins=10, title='Time To Eruption Distribution')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:24:04.734753Z","iopub.execute_input":"2022-01-25T12:24:04.735462Z","iopub.status.idle":"2022-01-25T12:24:05.651979Z","shell.execute_reply.started":"2022-01-25T12:24:04.735423Z","shell.execute_reply":"2022-01-25T12:24:05.65127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(\n      train,\n      y='time_to_eruption',\n      width=800, height=500,  title='Time To Eruption Distribution')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:26:04.699091Z","iopub.execute_input":"2022-01-25T12:26:04.699464Z","iopub.status.idle":"2022-01-25T12:26:04.761321Z","shell.execute_reply.started":"2022-01-25T12:26:04.699439Z","shell.execute_reply":"2022-01-25T12:26:04.760908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train,Test Set 분석\n\n### Train 셋","metadata":{}},{"cell_type":"code","source":"sensors = set()\nobservations = set()\nnan_columns = list()\nmissed_groups = list()\nfor_df = list()\n\nfor item in train_csvs:\n    name = int(item.split('.')[-2].split('/')[-1]) #train 폴더 csv 파일명 이름만 갖고오\n    at_least_one_missed = 0\n    frag = pd.read_csv(item)\n    missed_group = list()\n    missed_percents = list()\n    for col in frag.columns:\n        missed_percents.append(frag[col].isnull().sum() / len(frag))\n        if pd.isnull(frag[col]).all() == True:\n            at_least_one_missed = 1\n            nan_columns.append(col)\n            missed_group.append(col)\n    if len(missed_group) > 0:\n        missed_groups.append(missed_group)\n    sensors.add(len(frag.columns))\n    observations.add(len(frag))\n    for_df.append([name, at_least_one_missed] + missed_percents)\n    \nprint('고유한(Unique) 센서 수: ', sensors)\nprint('고유한(Unique) 행(row) 수: ', observations)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:28:46.934087Z","iopub.execute_input":"2022-01-25T12:28:46.934488Z","iopub.status.idle":"2022-01-25T12:34:57.623037Z","shell.execute_reply.started":"2022-01-25T12:28:46.93446Z","shell.execute_reply":"2022-01-25T12:34:57.622103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('센서값 Null인 센서수:',len(nan_columns))\n\nabsent_sensors = dict()\n\nfor item in nan_columns:\n    if item in absent_sensors:\n        absent_sensors[item]+=1\n    else:\n        absent_sensors[item]=0","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:37:00.709761Z","iopub.execute_input":"2022-01-25T12:37:00.710415Z","iopub.status.idle":"2022-01-25T12:37:00.71589Z","shell.execute_reply.started":"2022-01-25T12:37:00.710387Z","shell.execute_reply":"2022-01-25T12:37:00.71542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"absent_df = pd.DataFrame(absent_sensors.items(),columns=['Sensor','Missed sensors'])\n\nfig = px.bar(absent_df,\n            x='Sensor',\n            y='Missed sensors',\n            width=800,\n            height=500,\n            title='Number of missed sensors in training dataset')\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:38:25.68859Z","iopub.execute_input":"2022-01-25T12:38:25.689202Z","iopub.status.idle":"2022-01-25T12:38:25.771635Z","shell.execute_reply.started":"2022-01-25T12:38:25.68916Z","shell.execute_reply":"2022-01-25T12:38:25.77094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test Set","metadata":{}},{"cell_type":"code","source":"sensors = set()\nobservations = set()\nnan_columns = list()\nmissed_groups = list()\nfor_test_df = list()\n\nfor item in test_csvs:\n    name = int(item.split('.')[-2].split('/')[-1])\n    at_least_one_missed = 0\n    frag = pd.read_csv(item)\n    missed_group = list()\n    missed_percents = list()\n    for col in frag.columns:\n        missed_percents.append(frag[col].isnull().sum() / len(frag))\n        if pd.isnull(frag[col]).all() == True:\n            at_least_one_missed = 1\n            nan_columns.append(col)\n            missed_group.append(col)\n    if len(missed_group) > 0:\n        missed_groups.append(missed_group)\n    sensors.add(len(frag.columns))\n    observations.add(len(frag))\n    for_test_df.append([name, at_least_one_missed] + missed_percents)\n    \nprint('고유한(Unique) 센서 수: ', sensors)\nprint('고유한(Unique) 행(row) 수: ', observations)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:42:51.918357Z","iopub.execute_input":"2022-01-25T12:42:51.91872Z","iopub.status.idle":"2022-01-25T12:49:07.419195Z","shell.execute_reply.started":"2022-01-25T12:42:51.918693Z","shell.execute_reply":"2022-01-25T12:49:07.418425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('값이 비어있는(missed) 센서 수:', len(nan_columns))\n\nabsent_sensors = dict()\n\nfor item in nan_columns:\n    if item in absent_sensors:\n        absent_sensors[item] += 1\n    else:\n        absent_sensors[item] = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"absent_df = pd.DataFrame(absent_sensors.items(), columns=['Sensor', 'Missed sensors'])\n\nfig = px.bar(\n    absent_df, \n    x=\"Sensor\",\n    y='Missed sensors',\n    width=800,\n    height=500,\n    title='Number of missed sensors in test dataset'\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T12:51:51.298843Z","iopub.execute_input":"2022-01-25T12:51:51.299603Z","iopub.status.idle":"2022-01-25T12:51:51.354265Z","shell.execute_reply.started":"2022-01-25T12:51:51.299545Z","shell.execute_reply":"2022-01-25T12:51:51.353566Z"},"trusted":true},"execution_count":null,"outputs":[]}]}