{"cells":[{"metadata":{"_uuid":"6cfb979f25da614a4d1b204eb5d5550dbd9fc995"},"cell_type":"markdown","source":"The training data consists of 800,000 measurements per signal, for 20 miliseconds. In this notebook, I reduce the 800,000 metrics to 400 metrics per signal, by averaging out 2000 signals at a time. The model will be build on top of the output of the transformed training data."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a84673e3a23748554fed6c3de5380ff088d125c6"},"cell_type":"code","source":"op_dict = {}\nn_agg = 2000\n\n\nloop = int(round(8712/1000))\nfor counter in range(loop):\n    st = counter * 1000\n    if(((counter+1) * 1000) > 8712):\n        en = 8712\n    else:\n        en = (counter+1)*1000\n    start = 0\n    end = n_agg\n    train = pq.read_pandas('../input/train.parquet', columns=[str(i) for i in range(st,en)]).to_pandas()\n    agg_measure_list = []\n    print(\"column start = \"+ str(st)+\" column end = \"+str(en))\n    for loc2 in range(400):\n        mn = list(train.loc[start:end].mean(axis = 0,skipna = True))\n        agg_measure_list.append(mn)\n        start += n_agg\n        end += n_agg\n    print(\"List length = \"+ str(len(agg_measure_list)))\n    op_dict[str(counter)] = agg_measure_list\n    print(\"dict length = \"+ str(len(op_dict)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bdc393dddf631866ac845cc9f9569e4b1269acb"},"cell_type":"code","source":"df0 = pd.DataFrame(op_dict[\"0\"])\ndf0.columns = [list(range(0,1000))]\ndf1 = pd.DataFrame(op_dict[\"1\"])\ndf1.columns = [list(range(1000,2000))]\ndf2 = pd.DataFrame(op_dict[\"2\"])\ndf2.columns = [list(range(2000,3000))]\ndf3 = pd.DataFrame(op_dict[\"3\"])\ndf3.columns = [list(range(3000,4000))]\ndf4 = pd.DataFrame(op_dict[\"4\"])\ndf4.columns = [list(range(4000,5000))]\ndf5 = pd.DataFrame(op_dict[\"5\"])\ndf5.columns = [list(range(5000,6000))]\ndf6 = pd.DataFrame(op_dict[\"6\"])\ndf6.columns = [list(range(6000,7000))]\ndf7 = pd.DataFrame(op_dict[\"7\"])\ndf7.columns = [list(range(7000,8000))]\ndf8 = pd.DataFrame(op_dict[\"8\"])\ndf8.columns = [list(range(8000,8712))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d44bdf7624f50c4a8f4b7de4582e7531aa08088e"},"cell_type":"code","source":"df = pd.concat([df0, df1, df2, df3, df4, df5, df6, df7, df8], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb69c28155d3c77b02b8d3612c3b6db3d5bca5ea"},"cell_type":"code","source":"plt.plot(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9aa20019a1e8d110cfb60c8616c53a8576f18b90"},"cell_type":"code","source":"df.to_csv('train_compressed.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f0cc23aa16829999f858ca2920be7ca30a08791"},"cell_type":"code","source":"test = pq.read_pandas('../input/test.parquet', columns=[str(i) for i in range(8712,8715)]).to_pandas()\ntest_meta = pd.read_csv(\"../input/metadata_test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0441bcf496fd58636b1b97a60496d06971ed6c52"},"cell_type":"code","source":"start_loc = 8712\nend_loc = 8712 + 20337","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0066aa4aac3f3467cf53ec76e1ae21ba80dcb8f"},"cell_type":"code","source":"op_dict = {}\nn_agg = 2000\nimport math\n\nloop = int(math.ceil(20337/2000))\nfor counter in range(loop):\n    st = start_loc + (counter * 2000)\n    if(((counter+1) * 2000) > end_loc):\n        en = start_loc + end_loc\n    else:\n        en = start_loc + ((counter+1)*2000)\n    start = 0\n    end = n_agg\n    train = pq.read_pandas('../input/test.parquet', columns=[str(i) for i in range(st,en)]).to_pandas()\n    agg_measure_list = []\n    print(\"column start = \"+ str(st)+\" column end = \"+str(en))\n    for loc2 in range(400):\n        mn = list(train.loc[start:end].mean(axis = 0,skipna = True))\n        agg_measure_list.append(mn)\n        start += n_agg\n        end += n_agg\n    print(\"List length = \"+ str(len(agg_measure_list)))\n    op_dict[str(counter)] = agg_measure_list\n    print(\"dict length = \"+ str(len(op_dict)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26be47ba2445b4ded843bb22e3b052d456b7b445"},"cell_type":"code","source":"df0 = pd.DataFrame(op_dict[\"0\"])\ndf0.columns = [list(range(8712,10712))]\ndf1 = pd.DataFrame(op_dict[\"1\"])\ndf1.columns = [list(range(10712,12712))]\ndf2 = pd.DataFrame(op_dict[\"2\"])\ndf2.columns = [list(range(12712,14712))]\ndf3 = pd.DataFrame(op_dict[\"3\"])\ndf3.columns = [list(range(14712,16712))]\ndf4 = pd.DataFrame(op_dict[\"4\"])\ndf4.columns = [list(range(16712,18712))]\ndf5 = pd.DataFrame(op_dict[\"5\"])\ndf5.columns = [list(range(18712,20712))]\ndf6 = pd.DataFrame(op_dict[\"6\"])\ndf6.columns = [list(range(20712,22712))]\ndf7 = pd.DataFrame(op_dict[\"7\"])\ndf7.columns = [list(range(22712,24712))]\ndf8 = pd.DataFrame(op_dict[\"8\"])\ndf8.columns = [list(range(24712,26712))]\ndf9 = pd.DataFrame(op_dict[\"9\"])\ndf9.columns = [list(range(26712,28712))]\ndf10 = pd.DataFrame(op_dict[\"10\"])\ndf10.columns = [list(range(28712,29049))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1641d293f841a53cc413244e0fce704149a1e14a"},"cell_type":"code","source":"df = pd.concat([df0, df1, df2, df3, df4, df5, df6, df7, df8,df9,df10], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40638f68d0853af9dd80018638324f80f4a8742b"},"cell_type":"code","source":"test_meta = pd.read_csv(\"../input/metadata_test.csv\")\nmax(test_meta['signal_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"921df50d3f71a69c57ecd2748f6e684d6ee626f5"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76a8d86b0bebf628951e79d2110d73203e1762ad"},"cell_type":"code","source":"df.to_csv('test_compressed.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d15c0e341acb9ed4518560aa1b9b1e66258da2c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}