{"cells":[{"metadata":{"id":"of_OPrwmDP8a","colab_type":"code","colab":{},"_uuid":"c7cf476715e7d03a8ca009bb7d197aa82bed8b39","trusted":true},"cell_type":"code","source":"# One feature constructing","execution_count":null,"outputs":[]},{"metadata":{"id":"JBHyPPQ_WFgv","colab_type":"code","colab":{},"trusted":true,"_uuid":"6324c132f45a6b8ed60f093482e73ed5463df648"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pyarrow.parquet as pq","execution_count":null,"outputs":[]},{"metadata":{"id":"n50x4CF1rap8","colab_type":"code","colab":{},"trusted":true,"_uuid":"a8e36b9f7ca6c0276015e148f2d769dae6ae6e11"},"cell_type":"code","source":"metatrain=pd.read_csv ('../input/metadata_train.csv')\nm1i=metatrain[metatrain.target==1].index\nm0i=metatrain[metatrain.target==0].index","execution_count":null,"outputs":[]},{"metadata":{"id":"93eTLN965PY0","colab_type":"code","colab":{},"trusted":true,"_uuid":"aef07928bcfd07550e082b58545bc49125175c76"},"cell_type":"code","source":"\nsuin0 = pq.read_pandas('../input/train.parquet', \\\n                        columns=[str(i) for i in m0i [:500]]).to_pandas()\nsuin0=suin0.transpose ()\nsuin0.index=suin0.index.astype ('int')\nsuin1 = pq.read_pandas('../input/train.parquet', \\\n                        columns=[str(i) for i in m1i [:500]]).to_pandas()\nsuin1=suin1.transpose ()\nsuin1.index=suin1.index.astype ('int')","execution_count":null,"outputs":[]},{"metadata":{"id":"3_tfU0hoaPiK","colab_type":"code","colab":{},"trusted":true,"_uuid":"e46649f65d704393e7696ef3e92c46725dfaf33a"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom scipy.fftpack import fft,dct,idct,dstn,idstn\nfrom scipy.signal import find_peaks, peak_widths\nfrom scipy.ndimage import maximum_filter1d\n","execution_count":null,"outputs":[]},{"metadata":{"id":"vDEO2JmUW6or","colab_type":"code","colab":{},"trusted":true,"_uuid":"5069425ad90031b3d19f611ec93a369e2abcafed"},"cell_type":"code","source":"# This procedure shifts the maximum of the signal to zero\ndef shift1 (v):\n  nn=len (v)\n  s1=np.argmax (v); s4=s1\n  s2=set(find_peaks (v)[0])\n  s2.difference_update({s1})\n  for i2 in s2:\n    if v [i2]>0.85*v [s1]:\n      if (s1-i2) % nn < nn/6:\n        s3=((s1-i2) % nn)//2\n        s4=(i2+s3) % nn\n      elif (i2-s1) % nn < nn/6:\n        s3=((i2-s1) % nn)//2\n        s4=(s1+s3) % nn      \n  s4=s4.astype ('int') \n  return np.roll (v,-s4),s4\n\n# This procedure finds the difference between the signal and its approximation\n# using scipy.fftpack.dct (also shift1 used)\ndef redim (vec):\n  nidct=150\n  vec_trunc=np.zeros ([nidct])\n  vec_dct=dct (vec,n=800000,norm='ortho')\n  vec_nidct= idct (vec_dct,n=nidct, norm='ortho')\n  s1=np.argmax (vec_nidct)\n  vec_nidct_shift,s5=shift1 (vec_nidct)\n  vec_nidct_shift_dct =dct (vec_nidct_shift,norm='ortho')\n  vec_trunc[0:15]=vec_nidct_shift_dct [0:15]\n  vec_trunc_idct=idct (vec_trunc,n=800000, norm='ortho')\n  vec_shift=np.roll(vec,-s5*800000//nidct)\n  vec_dif=vec_shift-vec_trunc_idct\n  return vec_dif\n                ","execution_count":null,"outputs":[]},{"metadata":{"id":"oJf5RLfG4dsd","colab_type":"code","colab":{},"trusted":true,"_uuid":"3e85aab48426656eede94c7ff537f8ef98f3dd63"},"cell_type":"code","source":"# This procedure uses some filter and constructs some characteristic\n# for 15 rectangulars formed by the signal. Why so? Don't know. The best\n# parameters I found are here\ndef shtuk2 (vecvdi):\n  nv=5; nv1=3; nve=800//nv; ng=1000//nv1; nyc=nve*ng; nb=nyc//6\n  mat44=np.empty ([nv,nv1]); \n  for i1 in range (nv1):\n    for i0 in range(nv):\n      ve2=vecvdi.reshape (800,-1)[i0*nve:(i0+1)*nve,i1*ng:(i1+1)*ng].reshape (-1)\n      ve3=idct (ve2.reshape (-1),norm='ortho');\n      ve3s=ve3 [nb:]\n      ve4=maximum_filter1d(ve3s,58,mode='wrap')\n      ff=find_peaks (ve4,height=1,distance=2,width=1)\n      ff1=ff [1]['peak_heights'];\n      fs=ff1.sum ()/ff1.mean ()\n      mat44 [i0,i1]=fs\n  vem=np.min(mat44)\n  return vem","execution_count":null,"outputs":[]},{"metadata":{"id":"aUEjfGz4wH6b","colab_type":"code","colab":{},"trusted":true,"_uuid":"664b92ccd7d5f4658fb2720c53011fa0988f8705"},"cell_type":"code","source":"# We collect the values of shtuk2 procedure in two lists for 500 normal and \n# 500 fault signals\nmin0=[]; min1=[]; \nfor i in range(500):\n  no0=m0i[i]\n  ve0=suin0.loc [no0,:]\n  vdi0=redim (ve0)\n  chi0=shtuk2 (vdi0)\n  min0.append (chi0);\nfor i in range(500):  \n  no1=m1i[i]\n  ve1=suin1.loc [no1,:]\n  vdi1=redim (ve1)\n  chi1=shtuk2 (vdi1)\n  min1.append (chi1)\n  \n  ","execution_count":null,"outputs":[]},{"metadata":{"id":"rjrkvNyXBBvk","colab_type":"code","outputId":"51ad5086-b1d9-48cb-f0d6-7294f9ffa5e2","colab":{"base_uri":"https://localhost:8080/","height":69},"trusted":true,"_uuid":"ab1c5d30e9665a4e3e9263bbb96c023c31da7692"},"cell_type":"code","source":"# For this number ku we have that 153 of 500 faults are greater than ku, and \n# only 4 of 500 normal signals are greater than ku\nku=np.sort(min0)[-5]\naa=pd.DataFrame(min1); bb=pd.DataFrame (min0);\nprint ((bb>ku).sum(), (aa>ku).sum())","execution_count":null,"outputs":[]},{"metadata":{"id":"VgnPPPcW_K_X","colab_type":"code","outputId":"de97a10a-8b8e-4839-f59f-b9c5210dc3b8","colab":{"base_uri":"https://localhost:8080/","height":449},"_uuid":"ba378c879d6cf177eb0e43957db176dc1ebd329e"},"cell_type":"markdown","source":"# Histogram of two lists\n\n\n"},{"metadata":{"trusted":true,"_uuid":"5486968a587e0353631c90175399b22cc06c3872"},"cell_type":"code","source":"plt.title('500 normal vs 500 faults')\nplt.hist((min1,min0))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e337547fdccb1ee39df0696a49222917a89ec75b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"One feature.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}