{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import PIL\nimport os, re, time, tqdm\nimport numpy as np\nimport pandas as pd\nfrom sklearn import model_selection\nfrom collections import abc\nimport matplotlib.pyplot as plt\nimport cv2\nimport pydicom\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 512\nN_SHARDS_TRAIN = 15\nN_SHARDS_TEST = 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ndf_test = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def maybe_list(x):\n    if isinstance(x, list):\n        return x\n    elif isinstance(x,  abc.Iterable):\n        return list(x)\n    return [x]\n\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=maybe_list(value)))\n\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=maybe_list(value)))\n\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=maybe_list(value)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def serialize_example(pid, img, age, sex, smoking_status, fvc):\n    fvc = np.array(fvc)\n    feature = {\n        'patient_id':     _bytes_feature([pid]),\n        'image':          _bytes_feature([img]),\n        'age':            _int64_feature(age),\n        'sex':            _int64_feature(sex),\n        'smoking_status': _int64_feature(smoking_status),\n        'FVC':            _float_feature(fvc)          \n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir(\"../working/train\")\nos.mkdir(\"../working/test\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pids = os.listdir(\"../input/osic-pulmonary-fibrosis-progression/train/\")\nn = []\nfor pid in pids:\n    folder = os.path.join(\"../input/osic-pulmonary-fibrosis-progression/train/\", pid)\n    n.append(len(os.listdir(folder)))\nP = pd.DataFrame({'Patient': pids, 'n': n})\nP = P.merge(df_train[['Patient', 'Age', 'FVC']].groupby('Patient').mean(), on='Patient')\ngrouping = [(2, 'n'), (3, 'FVC')]\nfor N, c in grouping:\n    P['q_%s' % c] = np.array([(P[c] >= P[c].quantile(i / N)).astype('int') for i in range(1, N)]).sum(0)\nP['group'] = np.array([P['q_%s' % c] * 10 ** i for i, (_, c) in enumerate(grouping)]).sum(0)\nprint(\"%d groups\" % len(P.group.drop_duplicates())) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_shards = pd.DataFrame()\nfor shard, (_, idx) in enumerate(model_selection.StratifiedKFold(N_SHARDS_TRAIN, shuffle=True, random_state=42).split(P.Patient, P.group)):\n    shard_pids = P.Patient.iloc[idx]\n    print(\"Shard %d\" % shard)\n    df_train_shards = pd.concat([df_train_shards, pd.DataFrame({'shard': shard, 'Patient': shard_pids})])\n    examples = []\n    for pid in shard_pids:\n        folder = os.path.join(\"../input/osic-pulmonary-fibrosis-progression/train/\", pid)\n        i = 1\n        file = os.path.join(folder, \"%d.dcm\" % i)\n        fvc = df_train[df_train.Patient == pid].FVC\n        age = df_train[df_train.Patient == pid].Age.iloc[0]\n        smoking_status = df_train[df_train.Patient == pid].SmokingStatus.map({'Never smoked': 0, 'Ex-smoker': 1, 'Currently smokes': 2}).iloc[0]\n        sex = df_train[df_train.Patient == pid].Sex.map({'Female': 0, 'Male': 1}).iloc[0]\n        try:\n            while os.path.exists(file):    \n                di = pydicom.dcmread(file)\n                img = di.pixel_array\n                img = cv2.resize(img, (IMG_SIZE, IMG_SIZE)).reshape((IMG_SIZE, IMG_SIZE, 1))\n                img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n                i += 1\n                file = os.path.join(folder, \"%d.dcm\" % i)\n                example = serialize_example(str.encode(pid), img, age, sex, smoking_status, fvc)\n                examples.append(example)\n        except RuntimeError:\n            pass\n        pass\n    with tf.io.TFRecordWriter(os.path.join(\"../working/train\", \"train%.2d-%d.tfrec\" % (shard, len(examples)))) as writer:\n        for example in examples:\n            writer.write(example)\n            pass\n        pass\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xxx = df_train_shards.merge(df_train, on='Patient')\nc = 'FVC'\nplt.figure(figsize=(15, 10))\nfor shard in xxx.shard.drop_duplicates():\n    plt.subplot(3, 5, shard + 1)\n    plt.hist(xxx[xxx.shard == shard][c], bins=10, density=True, range=(xxx[c].min(), xxx[c].max()), alpha=0.5, histtype='step')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pids = os.listdir(\"../input/osic-pulmonary-fibrosis-progression/test/\")\nn = []\nfor pid in pids:\n    folder = os.path.join(\"../input/osic-pulmonary-fibrosis-progression/test/\", pid)\n    n.append(len(os.listdir(folder)))\nP = pd.DataFrame({'Patient': pids, 'n': n})\nP = P.merge(df_train[['Patient', 'Age', 'FVC']].groupby('Patient').mean(), on='Patient')\ngrouping = [(2, 'n'), (3, 'FVC')]\nfor N, c in grouping:\n    P['q_%s' % c] = np.array([(P[c] >= P[c].quantile(i / N)).astype('int') for i in range(1, N)]).sum(0)\nP['group'] = np.array([P['q_%s' % c] * 10 ** i for i, (_, c) in enumerate(grouping)]).sum(0)\nprint(\"%d groups\" % len(P.group.drop_duplicates())) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_shards = pd.DataFrame()\n# for shard, (_, idx) in enumerate(model_selection.StratifiedKFold(N_SHARDS_TEST, shuffle=True, random_state=42).split(P.Patient, P.group)):\nfor shard, (_, idx) in [(0, (None, list(range(P.shape[0]))))]:\n    shard_pids = P.Patient.iloc[idx]\n    print(\"Shard %d\" % shard)\n    df_test_shards = pd.concat([df_test_shards, pd.DataFrame({'shard': shard, 'Patient': shard_pids})])\n    examples = []\n    for pid in shard_pids:\n        folder = os.path.join(\"../input/osic-pulmonary-fibrosis-progression/test/\", pid)\n        i = 1\n        file = os.path.join(folder, \"%d.dcm\" % i)\n        fvc = df_test[df_test.Patient == pid].FVC\n        age = df_test[df_test.Patient == pid].Age.iloc[0]\n        smoking_status = df_test[df_test.Patient == pid].SmokingStatus.map({'Never smoked': 0, 'Ex-smoker': 1, 'Currently smokes': 2}).iloc[0]\n        sex = df_test[df_test.Patient == pid].Sex.map({'Female': 0, 'Male': 1}).iloc[0]\n        try:\n            while os.path.exists(file):    \n                di = pydicom.dcmread(file)\n                img = di.pixel_array\n                img = cv2.resize(img, (IMG_SIZE, IMG_SIZE)).reshape((IMG_SIZE, IMG_SIZE, 1))\n                img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n                i += 1\n                file = os.path.join(folder, \"%d.dcm\" % i)\n                example = serialize_example(str.encode(pid), img, age, sex, smoking_status, fvc)\n                examples.append(example)\n        except RuntimeError:\n            pass\n        pass\n    with tf.io.TFRecordWriter(os.path.join(\"../working/test\", \"test%.2d-%d.tfrec\" % (shard, len(examples)))) as writer:\n        for example in examples:\n            writer.write(example)\n            pass\n        pass\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_shards.to_csv(\"train_shards.csv\", index=False)\ndf_test_shards.to_csv(\"test_shards.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}