{
  "id": 472853,
  "title": "HMS - Harmful Brain Activity Classification 数据预处理",
  "url": "/competitions/hms-harmful-brain-activity-classification/discussion/472853",
  "author_name": "chenyushengxxxx",
  "post_date": "2024-02-02T11:51:22.192000",
  "votes": 3,
  "comment_count": 0,
  "views": 0,
  "content": "<p>import numpy as np<br>\nimport pandas as pd<br>\nimport seaborn as sns<br>\nfrom tqdm import tqdm<br>\nimport matplotlib.pyplot as plt<br>\nfrom IPython.display import Image, display, HTML<br>\nfrom PIL import Image<br>\nimport pywt<br>\nimport cv2<br>\nimport matplotlib as mpl<br>\nimport os<br>\nfrom sklearn.preprocessing import MinMaxScaler</p>\n<p>import winshell<br>\nfrom get_data import split_train_val_test</p>\n<p>pairing = {<br>\n        \"Fp1\": \"F7\",<br>\n        \"F7\": \"T3\",<br>\n        \"T3\": \"T5\",<br>\n        \"T5\": \"O1\",<br>\n        \"Fp2\": \"F8\",<br>\n        \"F8\": \"T4\",<br>\n        \"T4\": \"T6\",<br>\n        \"T6\": \"O2\",<br>\n        \"Fp1\": \"F3\",<br>\n        \"F3\": \"C3\",<br>\n        \"C3\": \"P3\",<br>\n        \"P3\": \"O1\",<br>\n        \"Fp2\": \"F4\",<br>\n        \"F4\": \"C4\",<br>\n        \"C4\": \"P4\",<br>\n        \"P4\": \"O2\",<br>\n        \"Fz\": \"Cz\",<br>\n        \"Cz\": \"Pz\",<br>\n        \"EKG\": '-'<br>\n    }</p>\n<p>def maddest(d, axis=None):<br>\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)</p>\n<p>def denoise(x, wavelet='haar', level=1):<br>\n    ret = {key: [] for key in x.columns}<br>\n    for pos in x.columns:<br>\n        coeff = pywt.wavedec(x[pos], wavelet, mode=\"per\")<br>\n        sigma = (1 / 0.6745) * maddest(coeff[-level])<br>\n        uthresh = sigma * np.sqrt(2 * np.log(len(x)))<br>\n        coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])<br>\n        ret[pos] = pywt.waverec(coeff, wavelet, mode='per')</p>\n<pre><code> pd.DataFrame(ret)\n</code></pre>\n<p>def plot_difference(data,is_sub,label_id):<br>\n    ax = None<br>\n    data = denoise(data, wavelet='db8')<br>\n    if is_sub:<br>\n        w = 5<br>\n        h = 20<br>\n    else:<br>\n        w = 50<br>\n        h = 20</p>\n<pre><code>plt.figure(figsize=(w, h),dpi = )\n i, (channel1, channel2)  (pairing.items()):\n    plt.subplot((pairing), , i + )\n     channel1==:\n        difference = data[channel1]\n    :\n        difference = data[channel1] - data[channel2]\n\n    plt.axis()\n    ax = sns.lineplot(data=difference)\n    \n    \n    \n\nplt.tight_layout()\nplt.savefig(+++ (label_id)+)\n\n</code></pre>\n<p>def get_spectrogram_part_image(spectrogram_sample_part):<br>\n    scaler = MinMaxScaler()<br>\n    spectrogram_sample_part = scaler.fit_transform(spectrogram_sample_part)<br>\n    spectrogram_sample_part = spectrogram_sample_part * 255  # 乘以255以将范围从[0-1]变成[0-255]<br>\n    img = Image.fromarray(np.uint8(spectrogram_sample_part))</p>\n<pre><code> img\n</code></pre>\n<p>def is_image_corrupted(img_path):<br>\n    try:<br>\n        img = Image.open(img_path)<br>\n        img.verify()<br>\n        return False<br>\n    except:<br>\n        return True</p>\n<p>def gen_eeg_spectrogram_images(data,eeg_path,spectrogram_path):</p>\n<pre><code>file = (,)\n\n index  ((data)):\n    eeg_id = data.iloc[index, :][]\n    spectrogram_id = data.iloc[index, :][]\n    label_id = data.iloc[index, :][]\n\n    eeg_label_offset_seconds = data.iloc[index, :][]\n    spectrogram_label_offset_seconds = data.iloc[index, :][]\n\n    eeg_sample = pd.read_parquet(\n        )\n    spectrogram_sample = pd.read_parquet(\n        )\n\n     eeg_sample.isnull().values.()  spectrogram_sample.isnull().values.():\n        \n\n    eeg_sub_sample = eeg_sample.iloc[(eeg_label_offset_seconds) * :(eeg_label_offset_seconds + ) * , :]\n\n    spectrogram_sub_sample = spectrogram_sample.iloc[\n                             (spectrogram_label_offset_seconds):(spectrogram_label_offset_seconds + )]\n\n    eeg_sample = eeg_sample.iloc[(eeg_sample) //  - :(eeg_sample) //  + , :]\n\n    spectrogram_sample = spectrogram_sample.query()\n\n    plot_difference(eeg_sample, , label_id)\n    \n\n    spectrogram_sample_LL = spectrogram_sample.(regex=)\n    spectrogram_sample_RL = spectrogram_sample.(regex=)\n    spectrogram_sample_LP = spectrogram_sample.(regex=)\n    spectrogram_sample_RP = spectrogram_sample.(regex=)\n\n    spectrogram_sub_sample_LL = spectrogram_sub_sample.(regex=)\n    spectrogram_sub_sample_RL = spectrogram_sub_sample.(regex=)\n    spectrogram_sub_sample_LP = spectrogram_sub_sample.(regex=)\n    spectrogram_sub_sample_RP = spectrogram_sub_sample.(regex=)\n\n    \n    \n\n    eeg_sample = eeg_sample.reset_index(drop=)\n    spectrogram_sample_LL = spectrogram_sample_LL.reset_index(drop=)\n    spectrogram_sample_RL = spectrogram_sample_RL.reset_index(drop=)\n    spectrogram_sample_LP = spectrogram_sample_LP.reset_index(drop=)\n    spectrogram_sample_RP = spectrogram_sample_RP.reset_index(drop=)\n\n    spectrogram_sub_sample_LL = spectrogram_sub_sample_LL.reset_index(drop=)\n    spectrogram_sub_sample_RL = spectrogram_sub_sample_RL.reset_index(drop=)\n    spectrogram_sub_sample_LP = spectrogram_sub_sample_LP.reset_index(drop=)\n    spectrogram_sub_sample_RP = spectrogram_sub_sample_RP.reset_index(drop=)\n\n    \n    \n    \n    \n    \n    \n\n    spectrogram_sample_LL = spectrogram_sample_LL.sort_index(axis=)\n    spectrogram_sample_RL = spectrogram_sample_RL.sort_index(axis=)\n    spectrogram_sample_LP = spectrogram_sample_LP.sort_index(axis=)\n    spectrogram_sample_RP = spectrogram_sample_RP.sort_index(axis=)\n\n    spectrogram_sample_arr = np.hstack(\n        [spectrogram_sample_LL, spectrogram_sample_RL, spectrogram_sample_LP, spectrogram_sample_RP])\n\n    spectrogram_image = get_spectrogram_part_image(spectrogram_sample_arr)\n\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(spectrogram_image)\n    \n\n    plt.imsave(++ + (label_id) + , spectrogram_image)\n\n    spectrogram_image = Image.(++ + (label_id) + )\n    eeg_image = Image.(++ + (label_id) + )\n\n    \n    \n\n    spectrogram_image = spectrogram_image.resize((, ))\n\n    eeg_image = eeg_image.resize((, ))\n    eeg_image = eeg_image.convert()\n\n    \n    \n\n    spectrogram_image.save(spectrogram_path +  + (label_id) + )\n\n    eeg_image.save(eeg_path +  + (label_id) + )\n\n\n    os.remove(++ + (label_id) + )\n    os.remove(++ + (label_id) + )\n\n    file.write((label_id) + )\n    file.flush()\n    ( +  + (label_id) + ,  + (label_id) + )\n\nfile.close()\n</code></pre>\n<p>def mix_eeg_spectrogram_images(data,eeg_path,spectrogram_path,eeg_spectrogram_path):</p>\n<pre><code>file = (, )\n\n os.path.exists():\n     (, )  file:\n         line  file:\n            label_id_str = line.replace(, )\n            row = data.loc[data[] == (label_id_str)]\n            label_id = (label_id_str)\n\n             os.path.exists(eeg_path +  + (label_id) + )  os.path.exists(spectrogram_path +  + (label_id) + ):\n\n                eeg_image = Image.(eeg_path +  + (label_id) + )\n                spectrogram_image = Image.(spectrogram_path +  + (label_id) + )\n\n                spectrogram_arr = np.asarray(spectrogram_image)\n                eeg_image = eeg_image.convert()\n                eeg_arr = np.asarray(eeg_image)\n\n                new_channel = np.zeros((spectrogram_arr.shape[], spectrogram_arr.shape[]), dtype=np.uint8) + eeg_arr\n                result = np.dstack([spectrogram_arr[:, :, i]  i  ()] + [new_channel])\n\n                output_image = Image.fromarray(result)\n\n                output_image.save(eeg_spectrogram_path +  + (label_id) + )\n\n                (, eeg_spectrogram_path +  + (label_id) + )\n</code></pre>",
  "messages": [
    {
      "id": 2632507,
      "postDate": "2024-02-02T11:51:22.193Z",
      "content": "<p>import numpy as np<br>\nimport pandas as pd<br>\nimport seaborn as sns<br>\nfrom tqdm import tqdm<br>\nimport matplotlib.pyplot as plt<br>\nfrom IPython.display import Image, display, HTML<br>\nfrom PIL import Image<br>\nimport pywt<br>\nimport cv2<br>\nimport matplotlib as mpl<br>\nimport os<br>\nfrom sklearn.preprocessing import MinMaxScaler</p>\n<p>import winshell<br>\nfrom get_data import split_train_val_test</p>\n<p>pairing = {<br>\n        \"Fp1\": \"F7\",<br>\n        \"F7\": \"T3\",<br>\n        \"T3\": \"T5\",<br>\n        \"T5\": \"O1\",<br>\n        \"Fp2\": \"F8\",<br>\n        \"F8\": \"T4\",<br>\n        \"T4\": \"T6\",<br>\n        \"T6\": \"O2\",<br>\n        \"Fp1\": \"F3\",<br>\n        \"F3\": \"C3\",<br>\n        \"C3\": \"P3\",<br>\n        \"P3\": \"O1\",<br>\n        \"Fp2\": \"F4\",<br>\n        \"F4\": \"C4\",<br>\n        \"C4\": \"P4\",<br>\n        \"P4\": \"O2\",<br>\n        \"Fz\": \"Cz\",<br>\n        \"Cz\": \"Pz\",<br>\n        \"EKG\": '-'<br>\n    }</p>\n<p>def maddest(d, axis=None):<br>\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)</p>\n<p>def denoise(x, wavelet='haar', level=1):<br>\n    ret = {key: [] for key in x.columns}<br>\n    for pos in x.columns:<br>\n        coeff = pywt.wavedec(x[pos], wavelet, mode=\"per\")<br>\n        sigma = (1 / 0.6745) * maddest(coeff[-level])<br>\n        uthresh = sigma * np.sqrt(2 * np.log(len(x)))<br>\n        coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])<br>\n        ret[pos] = pywt.waverec(coeff, wavelet, mode='per')</p>\n<pre><code> pd.DataFrame(ret)\n</code></pre>\n<p>def plot_difference(data,is_sub,label_id):<br>\n    ax = None<br>\n    data = denoise(data, wavelet='db8')<br>\n    if is_sub:<br>\n        w = 5<br>\n        h = 20<br>\n    else:<br>\n        w = 50<br>\n        h = 20</p>\n<pre><code>plt.figure(figsize=(w, h),dpi = )\n i, (channel1, channel2)  (pairing.items()):\n    plt.subplot((pairing), , i + )\n     channel1==:\n        difference = data[channel1]\n    :\n        difference = data[channel1] - data[channel2]\n\n    plt.axis()\n    ax = sns.lineplot(data=difference)\n    \n    \n    \n\nplt.tight_layout()\nplt.savefig(+++ (label_id)+)\n\n</code></pre>\n<p>def get_spectrogram_part_image(spectrogram_sample_part):<br>\n    scaler = MinMaxScaler()<br>\n    spectrogram_sample_part = scaler.fit_transform(spectrogram_sample_part)<br>\n    spectrogram_sample_part = spectrogram_sample_part * 255  # 乘以255以将范围从[0-1]变成[0-255]<br>\n    img = Image.fromarray(np.uint8(spectrogram_sample_part))</p>\n<pre><code> img\n</code></pre>\n<p>def is_image_corrupted(img_path):<br>\n    try:<br>\n        img = Image.open(img_path)<br>\n        img.verify()<br>\n        return False<br>\n    except:<br>\n        return True</p>\n<p>def gen_eeg_spectrogram_images(data,eeg_path,spectrogram_path):</p>\n<pre><code>file = (,)\n\n index  ((data)):\n    eeg_id = data.iloc[index, :][]\n    spectrogram_id = data.iloc[index, :][]\n    label_id = data.iloc[index, :][]\n\n    eeg_label_offset_seconds = data.iloc[index, :][]\n    spectrogram_label_offset_seconds = data.iloc[index, :][]\n\n    eeg_sample = pd.read_parquet(\n        )\n    spectrogram_sample = pd.read_parquet(\n        )\n\n     eeg_sample.isnull().values.()  spectrogram_sample.isnull().values.():\n        \n\n    eeg_sub_sample = eeg_sample.iloc[(eeg_label_offset_seconds) * :(eeg_label_offset_seconds + ) * , :]\n\n    spectrogram_sub_sample = spectrogram_sample.iloc[\n                             (spectrogram_label_offset_seconds):(spectrogram_label_offset_seconds + )]\n\n    eeg_sample = eeg_sample.iloc[(eeg_sample) //  - :(eeg_sample) //  + , :]\n\n    spectrogram_sample = spectrogram_sample.query()\n\n    plot_difference(eeg_sample, , label_id)\n    \n\n    spectrogram_sample_LL = spectrogram_sample.(regex=)\n    spectrogram_sample_RL = spectrogram_sample.(regex=)\n    spectrogram_sample_LP = spectrogram_sample.(regex=)\n    spectrogram_sample_RP = spectrogram_sample.(regex=)\n\n    spectrogram_sub_sample_LL = spectrogram_sub_sample.(regex=)\n    spectrogram_sub_sample_RL = spectrogram_sub_sample.(regex=)\n    spectrogram_sub_sample_LP = spectrogram_sub_sample.(regex=)\n    spectrogram_sub_sample_RP = spectrogram_sub_sample.(regex=)\n\n    \n    \n\n    eeg_sample = eeg_sample.reset_index(drop=)\n    spectrogram_sample_LL = spectrogram_sample_LL.reset_index(drop=)\n    spectrogram_sample_RL = spectrogram_sample_RL.reset_index(drop=)\n    spectrogram_sample_LP = spectrogram_sample_LP.reset_index(drop=)\n    spectrogram_sample_RP = spectrogram_sample_RP.reset_index(drop=)\n\n    spectrogram_sub_sample_LL = spectrogram_sub_sample_LL.reset_index(drop=)\n    spectrogram_sub_sample_RL = spectrogram_sub_sample_RL.reset_index(drop=)\n    spectrogram_sub_sample_LP = spectrogram_sub_sample_LP.reset_index(drop=)\n    spectrogram_sub_sample_RP = spectrogram_sub_sample_RP.reset_index(drop=)\n\n    \n    \n    \n    \n    \n    \n\n    spectrogram_sample_LL = spectrogram_sample_LL.sort_index(axis=)\n    spectrogram_sample_RL = spectrogram_sample_RL.sort_index(axis=)\n    spectrogram_sample_LP = spectrogram_sample_LP.sort_index(axis=)\n    spectrogram_sample_RP = spectrogram_sample_RP.sort_index(axis=)\n\n    spectrogram_sample_arr = np.hstack(\n        [spectrogram_sample_LL, spectrogram_sample_RL, spectrogram_sample_LP, spectrogram_sample_RP])\n\n    spectrogram_image = get_spectrogram_part_image(spectrogram_sample_arr)\n\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(spectrogram_image)\n    \n\n    plt.imsave(++ + (label_id) + , spectrogram_image)\n\n    spectrogram_image = Image.(++ + (label_id) + )\n    eeg_image = Image.(++ + (label_id) + )\n\n    \n    \n\n    spectrogram_image = spectrogram_image.resize((, ))\n\n    eeg_image = eeg_image.resize((, ))\n    eeg_image = eeg_image.convert()\n\n    \n    \n\n    spectrogram_image.save(spectrogram_path +  + (label_id) + )\n\n    eeg_image.save(eeg_path +  + (label_id) + )\n\n\n    os.remove(++ + (label_id) + )\n    os.remove(++ + (label_id) + )\n\n    file.write((label_id) + )\n    file.flush()\n    ( +  + (label_id) + ,  + (label_id) + )\n\nfile.close()\n</code></pre>\n<p>def mix_eeg_spectrogram_images(data,eeg_path,spectrogram_path,eeg_spectrogram_path):</p>\n<pre><code>file = (, )\n\n os.path.exists():\n     (, )  file:\n         line  file:\n            label_id_str = line.replace(, )\n            row = data.loc[data[] == (label_id_str)]\n            label_id = (label_id_str)\n\n             os.path.exists(eeg_path +  + (label_id) + )  os.path.exists(spectrogram_path +  + (label_id) + ):\n\n                eeg_image = Image.(eeg_path +  + (label_id) + )\n                spectrogram_image = Image.(spectrogram_path +  + (label_id) + )\n\n                spectrogram_arr = np.asarray(spectrogram_image)\n                eeg_image = eeg_image.convert()\n                eeg_arr = np.asarray(eeg_image)\n\n                new_channel = np.zeros((spectrogram_arr.shape[], spectrogram_arr.shape[]), dtype=np.uint8) + eeg_arr\n                result = np.dstack([spectrogram_arr[:, :, i]  i  ()] + [new_channel])\n\n                output_image = Image.fromarray(result)\n\n                output_image.save(eeg_spectrogram_path +  + (label_id) + )\n\n                (, eeg_spectrogram_path +  + (label_id) + )\n</code></pre>",
      "rawMarkdown": "import numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display, HTML\nfrom PIL import Image\nimport pywt\nimport cv2\nimport matplotlib as mpl\nimport os\nfrom sklearn.preprocessing import MinMaxScaler\n\nimport winshell\nfrom get_data import split_train_val_test\n\npairing = {\n        \"Fp1\": \"F7\",\n        \"F7\": \"T3\",\n        \"T3\": \"T5\",\n        \"T5\": \"O1\",\n        \"Fp2\": \"F8\",\n        \"F8\": \"T4\",\n        \"T4\": \"T6\",\n        \"T6\": \"O2\",\n        \"Fp1\": \"F3\",\n        \"F3\": \"C3\",\n        \"C3\": \"P3\",\n        \"P3\": \"O1\",\n        \"Fp2\": \"F4\",\n        \"F4\": \"C4\",\n        \"C4\": \"P4\",\n        \"P4\": \"O2\",\n        \"Fz\": \"Cz\",\n        \"Cz\": \"Pz\",\n        \"EKG\": '-'\n    }\n\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):\n    ret = {key: [] for key in x.columns}\n    for pos in x.columns:\n        coeff = pywt.wavedec(x[pos], wavelet, mode=\"per\")\n        sigma = (1 / 0.6745) * maddest(coeff[-level])\n        uthresh = sigma * np.sqrt(2 * np.log(len(x)))\n        coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n        ret[pos] = pywt.waverec(coeff, wavelet, mode='per')\n\n    return pd.DataFrame(ret)\n\ndef plot_difference(data,is_sub,label_id):\n    ax = None\n    data = denoise(data, wavelet='db8')\n    if is_sub:\n        w = 5\n        h = 20\n    else:\n        w = 50\n        h = 20\n\n    plt.figure(figsize=(w, h),dpi = 100)\n    for i, (channel1, channel2) in enumerate(pairing.items()):\n        plt.subplot(len(pairing), 1, i + 1)\n        if channel1=='EKG':\n            difference = data[channel1]\n        else:\n            difference = data[channel1] - data[channel2]\n\n        plt.axis('off')\n        ax = sns.lineplot(data=difference)\n        # plt.title(f\"{channel1}-{channel2}\")\n        # plt.xlabel(\"Time\")\n        # plt.ylabel(\"Difference in Values\")\n\n    plt.tight_layout()\n    plt.savefig(r'F:\\data\\hms\\temp'+'\\\\'+'eeg_'+ str(label_id)+\".png\")\n    # plt.show()\n\n\ndef get_spectrogram_part_image(spectrogram_sample_part):\n    scaler = MinMaxScaler()\n    spectrogram_sample_part = scaler.fit_transform(spectrogram_sample_part)\n    spectrogram_sample_part = spectrogram_sample_part * 255  # 乘以255以将范围从[0-1]变成[0-255]\n    img = Image.fromarray(np.uint8(spectrogram_sample_part))\n\n    return img\n\ndef is_image_corrupted(img_path):\n    try:\n        img = Image.open(img_path)\n        img.verify()\n        return False\n    except:\n        return True\n\ndef gen_eeg_spectrogram_images(data,eeg_path,spectrogram_path):\n\n    file = open(r'F:\\data\\hms\\temp\\labels.txt','w')\n\n    for index in range(len(data)):\n        eeg_id = data.iloc[index, :]['eeg_id']\n        spectrogram_id = data.iloc[index, :]['spectrogram_id']\n        label_id = data.iloc[index, :]['label_id']\n\n        eeg_label_offset_seconds = data.iloc[index, :]['eeg_label_offset_seconds']\n        spectrogram_label_offset_seconds = data.iloc[index, :]['spectrogram_label_offset_seconds']\n\n        eeg_sample = pd.read_parquet(\n            fr\"F:\\download\\hms-harmful-brain-activity-classification\\train_eegs\\{eeg_id}.parquet\")\n        spectrogram_sample = pd.read_parquet(\n            rf'F:\\download\\hms-harmful-brain-activity-classification\\train_spectrograms\\{spectrogram_id}.parquet')\n\n        if eeg_sample.isnull().values.any() or spectrogram_sample.isnull().values.any():\n            continue\n\n        eeg_sub_sample = eeg_sample.iloc[int(eeg_label_offset_seconds) * 200:int(eeg_label_offset_seconds + 1) * 200, :]\n\n        spectrogram_sub_sample = spectrogram_sample.iloc[\n                                 int(spectrogram_label_offset_seconds):int(spectrogram_label_offset_seconds + 1)]\n\n        eeg_sample = eeg_sample.iloc[len(eeg_sample) // 2 - 1000:len(eeg_sample) // 2 + 1000, :]\n\n        spectrogram_sample = spectrogram_sample.query('time >= 0 and time<=599')\n\n        plot_difference(eeg_sample, False, label_id)\n        # plot_difference(eeg_sub_sample,True)\n\n        spectrogram_sample_LL = spectrogram_sample.filter(regex='LL')\n        spectrogram_sample_RL = spectrogram_sample.filter(regex='RL')\n        spectrogram_sample_LP = spectrogram_sample.filter(regex='LP')\n        spectrogram_sample_RP = spectrogram_sample.filter(regex='RP')\n\n        spectrogram_sub_sample_LL = spectrogram_sub_sample.filter(regex='LL')\n        spectrogram_sub_sample_RL = spectrogram_sub_sample.filter(regex='RL')\n        spectrogram_sub_sample_LP = spectrogram_sub_sample.filter(regex='LP')\n        spectrogram_sub_sample_RP = spectrogram_sub_sample.filter(regex='RP')\n\n        # print(spectrogram_sample_LP)\n        # print(spectrogram_sub_sample_LL)\n\n        eeg_sample = eeg_sample.reset_index(drop=True)\n        spectrogram_sample_LL = spectrogram_sample_LL.reset_index(drop=True)\n        spectrogram_sample_RL = spectrogram_sample_RL.reset_index(drop=True)\n        spectrogram_sample_LP = spectrogram_sample_LP.reset_index(drop=True)\n        spectrogram_sample_RP = spectrogram_sample_RP.reset_index(drop=True)\n\n        spectrogram_sub_sample_LL = spectrogram_sub_sample_LL.reset_index(drop=True)\n        spectrogram_sub_sample_RL = spectrogram_sub_sample_RL.reset_index(drop=True)\n        spectrogram_sub_sample_LP = spectrogram_sub_sample_LP.reset_index(drop=True)\n        spectrogram_sub_sample_RP = spectrogram_sub_sample_RP.reset_index(drop=True)\n\n        # print(eeg_sample.head(10))\n        # print(eeg_sub_sample.head(10))\n        # print(spectrogram_sample_LL.head(10))\n        # print(spectrogram_sample_RL.head(10))\n        # print(spectrogram_sample_LP.head(10))\n        # print(spectrogram_sample_RP.head(10))\n\n        spectrogram_sample_LL = spectrogram_sample_LL.sort_index(axis=1)\n        spectrogram_sample_RL = spectrogram_sample_RL.sort_index(axis=1)\n        spectrogram_sample_LP = spectrogram_sample_LP.sort_index(axis=1)\n        spectrogram_sample_RP = spectrogram_sample_RP.sort_index(axis=1)\n\n        spectrogram_sample_arr = np.hstack(\n            [spectrogram_sample_LL, spectrogram_sample_RL, spectrogram_sample_LP, spectrogram_sample_RP])\n\n        spectrogram_image = get_spectrogram_part_image(spectrogram_sample_arr)\n\n        plt.xticks([])\n        plt.yticks([])\n        plt.imshow(spectrogram_image)\n        # plt.show()\n\n        plt.imsave(r'F:\\data\\hms\\temp'+'\\\\'+'spectrogram_' + str(label_id) + \".png\", spectrogram_image)\n\n        spectrogram_image = Image.open(r'F:\\data\\hms\\temp'+'\\\\'+'spectrogram_' + str(label_id) + \".png\")\n        eeg_image = Image.open(r'F:\\data\\hms\\temp'+'\\\\'+'eeg_' + str(label_id) + '.png')\n\n        # if is_image_corrupted(r'F:\\data\\hms\\temp'+'\\\\'+'eeg_' + str(label_id) + '.png') or is_image_corrupted(r'F:\\data\\hms\\temp'+'\\\\'+'spectrogram_' + str(label_id) + \".png\"):\n        #     continue\n\n        spectrogram_image = spectrogram_image.resize((640, 640))\n\n        eeg_image = eeg_image.resize((640, 640))\n        eeg_image = eeg_image.convert('L')\n\n        # if index % 100 == 0:\n        #     winshell.recycle_bin().empty(confirm=False, show_progress=False, sound=False)\n\n        spectrogram_image.save(spectrogram_path + \"\\\\\" + str(label_id) + '.png')\n\n        eeg_image.save(eeg_path + \"\\\\\" + str(label_id) + '.png')\n\n\n        os.remove(r'F:\\data\\hms\\temp'+'\\\\'+'spectrogram_' + str(label_id) + \".png\")\n        os.remove(r'F:\\data\\hms\\temp'+'\\\\'+'eeg_' + str(label_id) + '.png')\n\n        file.write(str(label_id) + '\\n')\n        file.flush()\n        print('finish: ' + 'spectrogram_' + str(label_id) + \".png\", 'eeg_' + str(label_id) + '.png')\n\n    file.close()\n\ndef mix_eeg_spectrogram_images(data,eeg_path,spectrogram_path,eeg_spectrogram_path):\n\n    file = open(r'F:\\data\\hms\\temp\\labels.txt', 'r')\n\n    if os.path.exists(r'F:\\data\\hms\\temp\\labels.txt'):\n        with open(r'F:\\data\\hms\\temp\\labels.txt', 'r') as file:\n            for line in file:\n                label_id_str = line.replace(\"\\n\", \"\")\n                row = data.loc[data['label_id'] == int(label_id_str)]\n                label_id = int(label_id_str)\n\n                if os.path.exists(eeg_path + '\\\\' + str(label_id) + '.png') and os.path.exists(spectrogram_path + '\\\\' + str(label_id) + '.png'):\n\n                    eeg_image = Image.open(eeg_path + '\\\\' + str(label_id) + '.png')\n                    spectrogram_image = Image.open(spectrogram_path + '\\\\' + str(label_id) + '.png')\n\n                    spectrogram_arr = np.asarray(spectrogram_image)\n                    eeg_image = eeg_image.convert('L')\n                    eeg_arr = np.asarray(eeg_image)\n\n                    new_channel = np.zeros((spectrogram_arr.shape[0], spectrogram_arr.shape[1]), dtype=np.uint8) + eeg_arr\n                    result = np.dstack([spectrogram_arr[:, :, i] for i in range(3)] + [new_channel])\n\n                    output_image = Image.fromarray(result)\n\n                    output_image.save(eeg_spectrogram_path + '\\\\' + str(label_id) + '.png')\n\n                    print('finish: ', eeg_spectrogram_path + '\\\\' + str(label_id) + '.png')\n",
      "votes": 3
    }
  ],
  "comments": [],
  "raw_markdown_by_id": {
    "2632507": "import numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display, HTML\nfrom PIL import Image\nimport pywt\nimport cv2\nimport matplotlib as mpl\nimport os\nfrom sklearn.preprocessing import MinMaxScaler\n\nimport winshell\nfrom get_data import split_train_val_test\n\npairing = {\n        \"Fp1\": \"F7\",\n        \"F7\": \"T3\",\n        \"T3\": \"T5\",\n        \"T5\": \"O1\",\n        \"Fp2\": \"F8\",\n        \"F8\": \"T4\",\n        \"T4\": \"T6\",\n        \"T6\": \"O2\",\n        \"Fp1\": \"F3\",\n        \"F3\": \"C3\",\n        \"C3\": \"P3\",\n        \"P3\": \"O1\",\n        \"Fp2\": \"F4\",\n        \"F4\": \"C4\",\n        \"C4\": \"P4\",\n        \"P4\": \"O2\",\n        \"Fz\": \"Cz\",\n        \"Cz\": \"Pz\",\n        \"EKG\": '-'\n    }\n\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):\n    ret = {key: [] for key in x.columns}\n    for pos in x.columns:\n        coeff = pywt.wavedec(x[pos], wavelet, mode=\"per\")\n        sigma = (1 / 0.6745) * maddest(coeff[-level])\n        uthresh = sigma * np.sqrt(2 * np.log(len(x)))\n        coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n        ret[pos] = pywt.waverec(coeff, wavelet, mode='per')\n\n    return pd.DataFrame(ret)\n\ndef plot_difference(data,is_sub,label_id):\n    ax = None\n    data = denoise(data, wavelet='db8')\n    if is_sub:\n        w = 5\n        h = 20\n    else:\n        w = 50\n        h = 20\n\n    plt.figure(figsize=(w, h),dpi = 100)\n    for i, (channel1, channel2) in enumerate(pairing.items()):\n        plt.subplot(len(pairing), 1, i + 1)\n        if channel1=='EKG':\n            difference = data[channel1]\n        else:\n            difference = data[channel1] - data[channel2]\n\n        plt.axis('off')\n        ax = sns.lineplot(data=difference)\n        # plt.title(f\"{channel1}-{channel2}\")\n        # plt.xlabel(\"Time\")\n        # plt.ylabel(\"Difference in Values\")\n\n    plt.tight_layout()\n    plt.savefig(r'F:\\data\\hms\\temp'+'\\\\'+'eeg_'+ str(label_id)+\".png\")\n    # plt.show()\n\n\ndef get_spectrogram_part_image(spectrogram_sample_part):\n    scaler = MinMaxScaler()\n    spectrogram_sample_part = scaler.fit_transform(spectrogram_sample_part)\n    spectrogram_sample_part = spectrogram_sample_part * 255  # 乘以255以将范围从[0-1]变成[0-255]\n    img = Image.fromarray(np.uint8(spectrogram_sample_part))\n\n    return img\n\ndef is_image_corrupted(img_path):\n    try:\n        img = Image.open(img_path)\n        img.verify()\n        return False\n    except:\n        return True\n\ndef gen_eeg_spectrogram_images(data,eeg_path,spectrogram_path):\n\n    file = open(r'F:\\data\\hms\\temp\\labels.txt','w')\n\n    for index in range(len(data)):\n        eeg_id = data.iloc[index, :]['eeg_id']\n        spectrogram_id = data.iloc[index, :]['spectrogram_id']\n        label_id = data.iloc[index, :]['label_id']\n\n        eeg_label_offset_seconds = data.iloc[index, :]['eeg_label_offset_seconds']\n        spectrogram_label_offset_seconds = data.iloc[index, :]['spectrogram_label_offset_seconds']\n\n        eeg_sample = pd.read_parquet(\n            fr\"F:\\download\\hms-harmful-brain-activity-classification\\train_eegs\\{eeg_id}.parquet\")\n        spectrogram_sample = pd.read_parquet(\n            rf'F:\\download\\hms-harmful-brain-activity-classification\\train_spectrograms\\{spectrogram_id}.parquet')\n\n        if eeg_sample.isnull().values.any() or spectrogram_sample.isnull().values.any():\n            continue\n\n        eeg_sub_sample = eeg_sample.iloc[int(eeg_label_offset_seconds) * 200:int(eeg_label_offset_seconds + 1) * 200, :]\n\n        spectrogram_sub_sample = spectrogram_sample.iloc[\n                                 int(spectrogram_label_offset_seconds):int(spectrogram_label_offset_seconds + 1)]\n\n        eeg_sample = eeg_sample.iloc[len(eeg_sample) // 2 - 1000:len(eeg_sample) // 2 + 1000, :]\n\n        spectrogram_sample = spectrogram_sample.query('time >= 0 and time<=599')\n\n        plot_difference(eeg_sample, False, label_id)\n        # plot_difference(eeg_sub_sample,True)\n\n        spectrogram_sample_LL = spectrogram_sample.filter(regex='LL')\n        spectrogram_sample_RL = spectrogram_sample.filter(regex='RL')\n        spectrogram_sample_LP = spectrogram_sample.filter(regex='LP')\n        spectrogram_sample_RP = spectrogram_sample.filter(regex='RP')\n\n        spectrogram_sub_sample_LL = spectrogram_sub_sample.filter(regex='LL')\n        spectrogram_sub_sample_RL = spectrogram_sub_sample.filter(regex='RL')\n        spectrogram_sub_sample_LP = spectrogram_sub_sample.filter(regex='LP')\n        spectrogram_sub_sample_RP = spectrogram_sub_sample.filter(regex='RP')\n\n        # print(spectrogram_sample_LP)\n        # print(spectrogram_sub_sample_LL)\n\n        eeg_sample = eeg_sample.reset_index(drop=True)\n        spectrogram_sample_LL = spectrogram_sample_LL.reset_index(drop=True)\n        spectrogram_sample_RL = spectrogram_sample_RL.reset_index(drop=True)\n        spectrogram_sample_LP = spectrogram_sample_LP.reset_index(drop=True)\n        spectrogram_sample_RP = spectrogram_sample_RP.reset_index(drop=True)\n\n        spectrogram_sub_sample_LL = spectrogram_sub_sample_LL.reset_index(drop=True)\n        spectrogram_sub_sample_RL = spectrogram_sub_sample_RL.reset_index(drop=True)\n        spectrogram_sub_sample_LP = spectrogram_sub_sample_LP.reset_index(drop=True)\n        spectrogram_sub_sample_RP = spectrogram_sub_sample_RP.reset_index(drop=True)\n\n        # print(eeg_sample.head(10))\n        # print(eeg_sub_sample.head(10))\n        # print(spectrogram_sample_LL.head(10))\n        # print(spectrogram_sample_RL.head(10))\n        # print(spectrogram_sample_LP.head(10))\n        # print(spectrogram_sample_RP.head(10))\n\n        spectrogram_sample_LL = spectrogram_sample_LL.sort_index(axis=1)\n        spectrogram_sample_RL = spectrogram_sample_RL.sort_index(axis=1)\n        spectrogram_sample_LP = spectrogram_sample_LP.sort_index(axis=1)\n        spectrogram_sample_RP = spectrogram_sample_RP.sort_index(axis=1)\n\n        spectrogram_sample_arr = np.hstack(\n            [spectrogram_sample_LL, spectrogram_sample_RL, spectrogram_sample_LP, spectrogram_sample_RP])\n\n        spectrogram_image = get_spectrogram_part_image(spectrogram_sample_arr)\n\n        plt.xticks([])\n        plt.yticks([])\n        plt.imshow(spectrogram_image)\n        # plt.show()\n\n        plt.imsave(r'F:\\data\\hms\\temp'+'\\\\'+'spectrogram_' + str(label_id) + \".png\", spectrogram_image)\n\n        spectrogram_image = Image.open(r'F:\\data\\hms\\temp'+'\\\\'+'spectrogram_' + str(label_id) + \".png\")\n        eeg_image = Image.open(r'F:\\data\\hms\\temp'+'\\\\'+'eeg_' + str(label_id) + '.png')\n\n        # if is_image_corrupted(r'F:\\data\\hms\\temp'+'\\\\'+'eeg_' + str(label_id) + '.png') or is_image_corrupted(r'F:\\data\\hms\\temp'+'\\\\'+'spectrogram_' + str(label_id) + \".png\"):\n        #     continue\n\n        spectrogram_image = spectrogram_image.resize((640, 640))\n\n        eeg_image = eeg_image.resize((640, 640))\n        eeg_image = eeg_image.convert('L')\n\n        # if index % 100 == 0:\n        #     winshell.recycle_bin().empty(confirm=False, show_progress=False, sound=False)\n\n        spectrogram_image.save(spectrogram_path + \"\\\\\" + str(label_id) + '.png')\n\n        eeg_image.save(eeg_path + \"\\\\\" + str(label_id) + '.png')\n\n\n        os.remove(r'F:\\data\\hms\\temp'+'\\\\'+'spectrogram_' + str(label_id) + \".png\")\n        os.remove(r'F:\\data\\hms\\temp'+'\\\\'+'eeg_' + str(label_id) + '.png')\n\n        file.write(str(label_id) + '\\n')\n        file.flush()\n        print('finish: ' + 'spectrogram_' + str(label_id) + \".png\", 'eeg_' + str(label_id) + '.png')\n\n    file.close()\n\ndef mix_eeg_spectrogram_images(data,eeg_path,spectrogram_path,eeg_spectrogram_path):\n\n    file = open(r'F:\\data\\hms\\temp\\labels.txt', 'r')\n\n    if os.path.exists(r'F:\\data\\hms\\temp\\labels.txt'):\n        with open(r'F:\\data\\hms\\temp\\labels.txt', 'r') as file:\n            for line in file:\n                label_id_str = line.replace(\"\\n\", \"\")\n                row = data.loc[data['label_id'] == int(label_id_str)]\n                label_id = int(label_id_str)\n\n                if os.path.exists(eeg_path + '\\\\' + str(label_id) + '.png') and os.path.exists(spectrogram_path + '\\\\' + str(label_id) + '.png'):\n\n                    eeg_image = Image.open(eeg_path + '\\\\' + str(label_id) + '.png')\n                    spectrogram_image = Image.open(spectrogram_path + '\\\\' + str(label_id) + '.png')\n\n                    spectrogram_arr = np.asarray(spectrogram_image)\n                    eeg_image = eeg_image.convert('L')\n                    eeg_arr = np.asarray(eeg_image)\n\n                    new_channel = np.zeros((spectrogram_arr.shape[0], spectrogram_arr.shape[1]), dtype=np.uint8) + eeg_arr\n                    result = np.dstack([spectrogram_arr[:, :, i] for i in range(3)] + [new_channel])\n\n                    output_image = Image.fromarray(result)\n\n                    output_image.save(eeg_spectrogram_path + '\\\\' + str(label_id) + '.png')\n\n                    print('finish: ', eeg_spectrogram_path + '\\\\' + str(label_id) + '.png')\n"
  }
}