{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.transforms import *\nfrom fastai.conv_learner import *\nfrom fastai.model import *\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"len(os.listdir(\"../input/test\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fdb78c4193fbbfc41cad69f6ca6ba6fad5c0417"},"cell_type":"code","source":"!ls ../input/train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53d71bf9f5aabc35a271d684c2518afa9d702ec5","collapsed":true},"cell_type":"code","source":"CATEGORIES=['Type_1','Type_2','Type_3']\ntrain_dir = '../input/train'\ntest_dir = '../input/test'\ntest_dir2 = '../input/test_stg2'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2da42d28073c916ad8b36c4e941978fade7c010"},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5f44d319296417103ed687e2a479511867ce367"},"cell_type":"code","source":"for category in CATEGORIES:\n    print('{} {} images'.format(category, len(os.listdir(os.path.join(train_dir, category)))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"754d963fb5d5d6caa7d34a546065adc3bae0c183","collapsed":true},"cell_type":"code","source":"len(os.listdir(test_dir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23253f9b30043727bace95df30d59882a75e3d56","scrolled":false},"cell_type":"code","source":"train = []\nfor  category in (CATEGORIES):\n    img=list(os.listdir(os.path.join(train_dir, category)))\n    for file in img:\n        if file.endswith('.jpg'):\n            train.append([file,category])\ntrain = pd.DataFrame(train, columns=['img','tag'])\n            \n        \n        #if file.endswith('.jpg'):\n            #print (file)\n            \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d580556d413bdf759790ebf2dd37432ce9f251c4"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8fcfff9ce941e7f4a7595b41a245687f9712e923"},"cell_type":"code","source":"train.to_csv('./cervical.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"660b26d78bb4f544ff25a17716f41bf070e7ae7e","collapsed":true},"cell_type":"code","source":"!mkdir ./train\n!mkdir ./test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0f5659fe2814d64487753f5214ce5e71efc3efb","collapsed":true},"cell_type":"code","source":"!ls ../input/train/Type_1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"374f365be1440c26ab9fae32609eb8da40600178","collapsed":true},"cell_type":"code","source":"from PIL import Image\nim=Image.open('../input/train/Type_3/1284.jpg')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a9d39b11d4689f5479e57e971d4cbf2a4738061","collapsed":true},"cell_type":"code","source":"im.mode","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb9334b03257137d244a4e6a7817760224bb816b","collapsed":true},"cell_type":"code","source":"im.resize((512,512))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1f6f74bfde76fc7696b47116db29437783d8d55","collapsed":true},"cell_type":"code","source":"im='../input/train/Type_1/513.jpg'\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b38f7d61c2a98392e08bf20b5c6d611f76f055e"},"cell_type":"markdown","source":"## Lets resize all our images and then save them"},{"metadata":{"trusted":true,"_uuid":"c27021dbc04ee60b350a0c0c3609bc5fb5e3ba2f","scrolled":true},"cell_type":"code","source":"#First train\nfrom scipy.misc import imread, imsave, imresize\nfrom PIL import Image\nfrom matplotlib import pyplot as plt\n%matplotlib inline\nimport cv2\nfrom fastai import transforms\n\nfor  category in (CATEGORIES):\n    source = os.listdir(os.path.join(train_dir, category))\n    source2=f'{train_dir}/{category}/'\n    img=list(source)\n    destination=\"./train/\"\n    for files in img:\n        if files.endswith('.jpg'):\n            im= cv2.imread(f'{source2}{files}')\n            \n            print(f'{source2}{files}')\n            b,g,r = cv2.split(im)\n            im2 = cv2.merge([r,g,b])\n            im3 = Image.fromarray(transforms.scale_min(im2,512))\n            im3.save(f'{destination}{files}')\n        \n#try:\n        \n      \n#except (ValueError):\n     #print(f'{source2}{files}')\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1d763bca416515f742ac8aef837448ce2bdeda0"},"cell_type":"code","source":"len(os.listdir('./train'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"de812b9f4b3cf7fd64b772b1704dcaea9076f633"},"cell_type":"code","source":"#Now Test\nfrom scipy.misc import imread, imsave, imresize\nfrom PIL import Image\nfrom matplotlib import pyplot as plt\n%matplotlib inline\nimport cv2\nfrom fastai import transforms\n\nsource = os.listdir(os.path.join(test_dir))\nsource2=f'{test_dir}/'\nimg=list(source)\ndestination=\"./test/\"\nfor files in img:\n    if files.endswith('.jpg'):\n        im= cv2.imread(f'{source2}{files}')\n        print(f'{source2}{files}')\n        b,g,r = cv2.split(im)\n        im2 = cv2.merge([r,g,b])\n        im3 = Image.fromarray(transforms.scale_min(im2,512))\n        im3.save(f'{destination}{files}')\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77fe2f5196349fd0273205f98bbcdef5bc2292c7"},"cell_type":"code","source":"#Now Test stage2\nfrom scipy.misc import imread, imsave, imresize\nfrom PIL import Image\nfrom matplotlib import pyplot as plt\n%matplotlib inline\nimport cv2\nfrom fastai import transforms\n\nsource = os.listdir(os.path.join(test_dir2))\nsource2=f'{test_dir2}/'\nimg=list(source)\ndestination=\"./test/\"\nfor files in img:\n    if files.endswith('.jpg'):\n        im= cv2.imread(f'{source2}{files}')\n        print(f'{source2}{files}')\n        b,g,r = cv2.split(im)\n        im2 = cv2.merge([r,g,b])\n        im3 = Image.fromarray(transforms.scale_min(im2,512))\n        im3.save(f'{destination}{files}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c22042f0a1fc5bd83b34e518e2dcbdeaf2814a9"},"cell_type":"code","source":"len(os.listdir('./test'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d5d931bc86f8898352698447941902f240a3a4d2"},"cell_type":"code","source":"img = cv2.imread('./test/12.jpg')\nimg2 = cv2.imread('../input/test/12.jpg')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71b98062c3630cadb6644e7266590c126456df63"},"cell_type":"code","source":"plt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b07d49e407cd9b5910aeacb281457c36dcb9a43"},"cell_type":"code","source":"train='./train'\ntest='./test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4654953ac849233065877ee386c200cb6b60e1b1"},"cell_type":"code","source":"label_csv = './cervical.csv'\nn = len(list(open(label_csv)))-2\nval_idxs = get_cv_idxs(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"706fd47a115f16a2700206d7d22274baa91a1f80"},"cell_type":"code","source":"PATH=''\narch=resnet34\nsz=128\ndata = ImageClassifierData.from_csv(PATH,train,label_csv,bs=10,val_idxs=val_idxs,tfms=tfms_from_model(arch, sz,max_zoom=1.1),test_name=test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dbd9f5ece717d9f1733128eaa13b9bcfdae94c37"},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57a47f3559ac1b3c5a66c94c249273e02b6d023e"},"cell_type":"code","source":"lrf=learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72e41beeea0936d9459d0b807e517a3df230ba6e"},"cell_type":"code","source":"learn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ded2043fa3740fa90ae5fdf6c9fe171aff3334ce"},"cell_type":"code","source":"lr=5e-5\nlearn.fit(lr,3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a884edfda73ebd5a4ab050399219d3e3ae4d591a"},"cell_type":"code","source":"lr=5e-3\nlearn.fit(lr,3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0e50e44229570b969bb30742c623ef6c396a9b1"},"cell_type":"code","source":"lr=5e-2\nlearn.fit(lr,3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"799b5ae0af3eb0f6127d6e604bbc9a9ab5f9083c"},"cell_type":"code","source":"lrs=np.array([lr/6,lr/3,lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0794bd4f80f4623fe419fb3710df63a95b655c64"},"cell_type":"code","source":"learn.unfreeze()\nlearn.fit(lrs,3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c277913d1dedde5888cb3782225a935b9cc55cce"},"cell_type":"code","source":"learn.fit(lrs,3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"56cda50ec21db8276aa3958f7bdc79504f891ae4"},"cell_type":"code","source":"def get_data(sz,bs):\n    \n    data=ImageClassifierData.from_csv(PATH,train,label_csv,bs,val_idxs=val_idxs,tfms=tfms_from_model(arch, sz,max_zoom=1.1),test_name=test)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"105115bbf7d34b7e4643108fe7be11191ce59dde"},"cell_type":"code","source":"learn.set_data(get_data(256,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"585fe229f9e0e05766d96f18abfd4eeaeefea9bc"},"cell_type":"code","source":"learn.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e727f7d6d6c0dc653ddc82e02ac2932ae20baa4"},"cell_type":"code","source":"lr=1e-4\nlearn.fit(lr,3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d213faef798b4b1567ba14557635b6d8cb39a09d"},"cell_type":"code","source":"log_preds, y = learn.TTA(is_test=True)\nprobs = np.mean(np.exp(log_preds),0)\nds=pd.DataFrame(probs)\nds.columns=data.classes\nds.insert(0,'id',[o.rsplit('/', 1)[1] for o in data.test_ds.fnames])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69f40c95b43f08f4e4bd64e3b1bfafe1e192e197"},"cell_type":"code","source":"#Check for validaton sample\nlog_preds, y = learn.TTA()\nprobs = np.mean(np.exp(log_preds),0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a049d7a8f969b133f1dd781cfbea125d12551ad"},"cell_type":"code","source":"accuracy_np(probs,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a725b7afbc254eab3bfa16dca990e9435e107b21"},"cell_type":"code","source":"# import the modules we'll need\nfrom IPython.display import HTML\nimport pandas as pd\nimport numpy as np\nimport base64\n\n# function that takes in a dataframe and creates a text link to  \n# download it (will only work for files < 2MB or so)\ndef create_download_link(df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv()\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\n# create a random sample dataframe\ndf = pd.DataFrame(np.random.randn(50, 4), columns=list('ABCD'))\n\n# create a link to download the dataframe\ncreate_download_link(ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"42de071213c47aea0f13ddc96d773e61d758bd01"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}