{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# imports\nimport matplotlib.pyplot as plt\nfrom scipy import signal\nimport seaborn as sns\nimport glob\nimport datetime\nimport os\nfrom tempfile import TemporaryFile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data\nDiractory = '../input/predict-volcanic-eruptions-ingv-oe'\ntrain = pd.read_csv(os.path.join(Diractory, 'train.csv'))\ntest_submission = pd.read_csv(os.path.join(Diractory, 'sample_submission.csv'))\n\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert time to eruption to days, hours, minutes\n#train['h:m:s'] = (train['time_to_eruption']\n#                  .apply(lambda x:datetime.timedelta(seconds = x/100)))\n#display train\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Getting sample ids\nsample_ids = train['segment_id'].values\n\nsample_ids\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Looking at every sigment_id file and cleaning the NaN values by replacing it with the mean of every sample.\n# Need to extend to every segment_id\n#segment_id = '1136037770'\n#segments = []\n# for i, segment_id in enumerate(sample_ids):\n#temp = pd.read_csv(os.path.join(Diractory, f'train/{segment_id}.csv')).T\n#temp_mean = temp.mean()\n#segments.append(temp.fillna(temp_mean).T)\n\n#segments","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# segment_id = '1136037770'\nsensor_1 = np.zeros((4431,60001))\nprint('preparing sensor_1')\nfor i, segment_id in enumerate(sample_ids):\n    temp = pd.read_csv(os.path.join(Diractory, f'train/{segment_id}.csv'))\n    sensor_1[i,:] = temp['sensor_10']\n    if (i % 200 == 0):\n        print(i)\n    #print(i)\n\nprint('done')\n# temp_mean = temp.mean()\n# segments.append(temp.fillna(temp_mean).T)\n#sensors[0,:] = temp['sensor_1']\n#sensors[0,:]\n#segments","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# time to eruption array\ntime_to_eruption = train['time_to_eruption'].values\nprint(time_to_eruption)\nprint(time_to_eruption.shape)\n#print()\n#print(time_to_eruption.size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# patial and empty data analysis for sensor_1\ncount_empty = 0\ncount_partial = 0\ni = 0\nsegments_id_to_remove = []\nfor segment_id in sensor_1:\n    count_Nan = np.count_nonzero(np.isnan(segment_id))\n    # print(count_Nan)\n    \n    if (count_Nan > 0) & (count_Nan < 60001):\n        print(count_Nan)\n        segments_id_to_remove.append(i)\n        \n        count_partial = count_partial + 1\n    if count_Nan == 60001:\n        print(count_Nan)\n        count_empty = count_empty + 1\n        segments_id_to_remove.append(i)\n        \n    i = i + 1\nprint('Number of empty files for sensor_1: ',count_empty)\nprint('Number of partial files for sensor_1:',count_partial)\n\n#we chose to delete all partial and empty segments for now,\n#as well as their time_to_eruption\n\n# print('Need to remove indices' + str(segments_id_to_remove))\nprint()\nsensor_1_clean = np.delete(sensor_1,segments_id_to_remove,axis=0)\nprint(sensor_1_clean)\nprint(\"sensor 1 clean shape: \"+str(sensor_1_clean.shape) + str(type(sensor_1_clean)))\nprint()\n#print(train)\n#remove title\n#time_to_eruption_clean1 = np.delete(time_to_eruption,0,axis=0)\n#print(time_to_eruption_clean1.shape)\n#print(train.to_numpy())\ntimes_and_ids_clean = np.delete(train.to_numpy(),segments_id_to_remove,axis=0)\nprint(times_and_ids_clean)\n#print(\"time to eruption clean shape: \" +str(time_to_eruption_clean.shape))\n# segments_ids_clean = np.delete(train['segment_id'].values,segments_id_to_remove,axis=0)\n# print(\"segment_ids clean shape: \" +str(segments_ids_clean.shape))\n\n#times_and_ids_clean = [segments_ids_clean,time_to_eruption_clean]\nprint(\"times and ids shape: \"+str(times_and_ids_clean.shape) + str(type(times_and_ids_clean)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# checking data is clean\ncount_empty = 0\ncount_partial = 0\ni = 0\nsegments_id_to_remove = []\nfor segment_id in sensor_1_clean:\n    count_Nan = np.count_nonzero(np.isnan(segment_id))\n    # print(count_Nan)\n    \n    if (count_Nan > 0) & (count_Nan < 60001):\n        print(count_Nan)\n        segments_id_to_remove.append(i)\n        \n        count_partial = count_partial + 1\n    if count_Nan == 60001:\n        print(count_Nan)\n        count_empty = count_empty + 1\n        segments_id_to_remove.append(i)\n        \n    i = i + 1\nprint('Number of empty files for sensor_1: ',count_empty)\nprint('Number of partial files for sensor_1:',count_partial)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save array to csv\n# os.path.join('../input/predict-volcanic-eruptions-ingv-oe')\n# np.savetxt('sensor_1_clean.csv', sensor_1_clean, delimiter=',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sensor_1_clean.shape\nsensor_1_clean.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# psd of sensor_1 cleaned\n\nPSD_sensor_1 = []\n\nfor i in range(sensor_1_clean.shape[0]):\n    #col_mean = df_example[col].mean()\n    #df_example[col].fillna(col_mean, inplace=True)\n    freq, psd = signal.welch(sensor_1_clean[i],100)\n    plt.loglog(freq,psd)\n    PSD_sensor_1.append(psd)\n    if i % 1000 == 0:\n        print(i)\nprint('PSD done')\nPSD_sensor_1 = np.asarray(PSD_sensor_1)\nprint(\"Shape of PSD: \"+ str(PSD_sensor_1.shape) + str(type(PSD_sensor_1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save psd and sensor_x_clean, time_to eruption to numpy file\nos.path.join('../output/kaggle/working/sensors_data')\nnp.save('sensor_10_time_to_eruption', times_and_ids_clean)\nos.path.join('../output/kaggle/working/sensors_data')\nnp.save('sensor_10_clean', sensor_1_clean)\nos.path.join('../output/kaggle/working/psd')\nnp.save('sensor_10_psd', PSD_sensor_1)\nos.path.join('../output/kaggle/working/freq')\nnp.save('freq_10', freq)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}