{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# 比赛 https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification\n\n# from IPython.core.interactiveshell import InteractiveShell\n# InteractiveShell.ast_node_interactivity = 'all'\n\nimport os\nimport sys\nimport gc\nimport math\nimport pickle\nimport random\nimport time\nimport psutil\nimport pytz\nfrom datetime import datetime\nfrom collections import defaultdict\nfrom contextlib import contextmanager\n\nimport warnings\nwarnings.filterwarnings('ignore')  # warnings.filterwarnings(action='once')\n\nfrom tqdm import tqdm, tqdm_notebook\n\nimport numpy as np\nimport pandas as pd\n_ = np.seterr(divide='ignore', invalid='ignore')\n\npd.set_option('display.max_columns', None)\n# pd.set_option('display.max_columns', 100)\npd.set_option('display.max_rows', None)\n# pd.set_option('display.max_rows', 100)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.style as style\nstyle.use('fivethirtyeight')\nimport seaborn as sns\n\n# 直接在cell中显示图片，支持jpg、png、jpeg等格式，Image('./2.JPG')\nfrom IPython.display import Image  \n\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\ndef show_process_mem_usage(info_str=''):    ## 显示当前进程占用内存大小\n    process = psutil.Process(os.getpid())\n    memory_usage = process.memory_info().rss\n    percent = psutil.virtual_memory().percent\n    \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    \n    if memory_usage >= 2.**30:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**30:.3f} GB, percentage: {percent:.2f}% 【{dt_str}】')\n    elif memory_usage >= 2.**20:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**20:.3f} MB, percentage: {percent:.2f}% 【{dt_str}】')\n    elif memory_usage >= 2.**10:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**10:.3f} KB, percentage: {percent:.2f}% 【{dt_str}】')\n    else:\n        print(f'{info_str} current process memory usage: {memory_usage} B, percentage: {percent:.2f}% 【{dt_str}】')\n\ndef logging(*info, file_name='./running_log.txt'):\n    log_info = ' '.join([str(s) for s in info])\n    with open(file_name, 'a') as f:\n        f.write(log_info + '\\n')\n\n@contextmanager\ndef trace(trace_msg):    ## 追踪内存变化和运行时间\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info()[0] / 2. ** 30\n    yield\n    m1 = p.memory_info()[0] / 2. ** 30\n    delta = m1 - m0\n    sign = '+' if delta >= 0 else '-'\n    delta = math.fabs(delta)\n    trace_msg = str(trace_msg)\n    \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    print(f\"[{m1:.3f}GB({sign}{delta:.3f}GB):{time.time() - t0:.3f}sec] {trace_msg} 【{dt_str}】\", file=sys.stdout)\n    \ndef seed_all(random_seed=42):\n    os.environ['PYTHONHASHSEED'] = str(random_seed)\n    random.seed(random_seed)\n    np.random.seed(random_seed)\n    tf.random.set_seed(random_seed)\n    torch.manual_seed(random_seed)\n    torch.cuda.manual_seed(random_seed)\n    torch.backends.cudnn.deterministic = True\n\ndef keepbusy(num=10000):\n    start_t = time.time()\n    for i in range(num):\n        ftpt(f'i: {i}, taken time: {time.time() - start_t:.7f}')\n        time.sleep(60)\n\ndef ftpt(msg = 'having run this cell'):  # foot_print   \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_string = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    print(f'{dt_string}: {msg}')\n    \n# 质数列表  [7, 53, 97, 317, 577, 997, 7753, 9973, 53113, 99991, 153133, 377171, 515371, 737353, 999983, 5157133, 7757537, 9999991, 99999989, 999999937]\nRANDOM_SEED = 53113\nseed_all(RANDOM_SEED)\n\ndataset_path = '../input/ranzcr-clip-catheter-line-classification'\nos.listdir(dataset_path)\nglobal_start_t = time.time()\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_PATH = '../input/ranzcr-clip-catheter-line-classification/train/'\nfile_names = os.listdir(TRAIN_PATH)\nlen(file_names)\nfile_names[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import cv2\n\n# total_t_1, WBB_t = 0, 0\n# for i, file_name in enumerate(file_names):\n#     file_path = TRAIN_PATH + file_name\n\n#     start_t_1 = time.time()\n#     image = cv2.imread(file_path)\n#     total_t_1 += time.time() - start_t_1\n    \n#     image = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)\n#     start_t_3 = time.time()\n#     mask = image > 0\n#     image = image[np.ix_(mask.any(1), mask.any(0))]\n#     WBB_t += time.time() - start_t_3\n    \n#     if i%200==0:\n#         print('file_name: ', file_name)\n#         print(f'total_t_1: {total_t_1:.5f}, WBB_t: {WBB_t:.5f}')\n    \n# print(f'Final total_t_1: {total_t_1:.5f}, WBB_t: {WBB_t:.5f}')\n\n############################################################################################\n\n# file_name:  1.2.826.0.1.3680043.8.498.77518942369171843789006015594910539466.jpg\n# total_t_1: 92.87874, WBB_t: 62.81078\n# file_name:  1.2.826.0.1.3680043.8.498.73171729950725506271487678682474036147.jpg\n# total_t_1: 103.78828, WBB_t: 69.73897","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\n\nOUTPUT_DIR = './train_data_wbb/'\n\nif os.path.exists(OUTPUT_DIR):\n    os.system('rm -rf ./train_data_wbb')\n\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"cnt = 0\nstart_t = time.time()\nfor i, file_name in enumerate(file_names):\n    file_path = TRAIN_PATH + file_name\n\n    image1 = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)\n#     image2 = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)\n    mask = image1 > 0\n    image2 = image1[np.ix_(mask.any(1), mask.any(0))]\n    \n    if image1.shape != image2.shape:\n        cnt += 1\n        target_file_path = OUTPUT_DIR + file_name\n        cv2.imwrite(target_file_path, image2) \n        \n    if i%500==0:\n        print(f'current i: {i}, cnt: {cnt}')\n    \nftpt(f'cnt: {cnt} total cost time: {time.time()-start_t:.3f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\ntarget_cols = test.iloc[:, 1:12].columns.tolist()\ntest[target_cols] = 0\ntest[['StudyInstanceUID'] + target_cols].to_csv('./submission.csv', index=False)\nprint('test.head() is ', test.head())\n\nprint(f'finished total cost time: {time.time()-global_start_t:.5f}')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}