{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"7064ba1c-6db3-fec8-2e0c-9c67c9159ea5"},"source":"AI for Screening\n================"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9572f5fb-74c7-bd85-db16-690ae73451f7"},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nimport glob\n\ntrain = glob.glob('../input/train/**/*.jpg') + glob.glob('../input/additional/**/*.jpg')\ntrain = pd.DataFrame([[p.split('/')[3],p.split('/')[4],p] for p in train], columns = ['type','image','path'])\n\ntest = glob.glob('../input/test/*.jpg')\ntest = pd.DataFrame([[p.split('/')[3],p] for p in test], columns = ['image','path'])\n\nsub = pd.read_csv('../input/sample_submission.csv')\nprint(len(train),len(test),len(sub))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a6bf536f-8277-fad4-ce32-ff832bbd708b"},"outputs":[],"source":"from PIL import ImageFilter, ImageStat, Image, ImageDraw\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport cv2\n\ntypes = train.groupby('type', as_index=False)['path'].count()\n_ = types.plot(kind='bar', x='type', y='path', figsize=(7,4))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"bdec5ec8-d8b6-9c81-3897-302c5c4c844e"},"outputs":[],"source":"types"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2f78adbb-2467-7f18-26dc-61622489f080"},"outputs":[],"source":"from multiprocessing import Pool, cpu_count\n\ndef im_multi(path):\n    try:\n        im_stats_im_ = Image.open(path)\n        return [path, {'size': im_stats_im_.size}]\n    except:\n        print(path)\n        return [path, {'size': [0,0]}]\n\ndef im_stats(im_stats_df):\n    im_stats_d = {}\n    p = Pool(cpu_count())\n    ret = p.map(im_multi, im_stats_df['path'])\n    for i in range(len(ret)):\n        im_stats_d[ret[i][0]] = ret[i][1]\n    im_stats_df['size'] = im_stats_df['path'].map(lambda x: ' '.join(str(s) for s in im_stats_d[x]['size']))\n    return im_stats_df\n\ntrain = im_stats(train)\nsizes = train.groupby('size', as_index=False)['path'].count()\n_ = sizes.plot(kind='bar', x='size', y='path', figsize=(7,4))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e9dc439e-3e27-a97c-aac7-1cf90d44b461"},"outputs":[],"source":"sizes"},{"cell_type":"markdown","metadata":{"_cell_guid":"5c799d7e-2d77-4c36-0b05-ec2bbabcb706"},"source":"Train and Test\n=============="},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2023476f-df2c-07e8-b2e0-3ca2f1a63ae1"},"outputs":[],"source":"from sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import log_loss\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom keras.models import Sequential\nfrom keras.layers.core import Dense, Dropout, Flatten\nfrom keras.layers.convolutional import Convolution2D, ZeroPadding2D, MaxPooling2D\nfrom keras import optimizers\n\ndef get_im_cv2(path):\n    img = cv2.imread(path)\n    resized = cv2.resize(img, (64, 64), cv2.INTER_LINEAR)\n    return [path, resized]\n\ndef normalize_image_features(paths):\n    imf_d = {}\n    p = Pool(cpu_count())\n    ret = p.map(get_im_cv2, paths)\n    for i in range(len(ret)):\n        imf_d[ret[i][0]] = ret[i][1]\n    ret = []\n    fdata = [imf_d[f] for f in paths]\n    fdata = np.array(fdata, dtype=np.uint8)\n    fdata = fdata.transpose((0, 3, 1, 2))\n    fdata = fdata.astype('float32')\n    fdata = fdata / 255\n    return fdata\n\ntrain = glob.glob('../input/train/**/*.jpg') #+ glob.glob('../input/additional/**/*.jpg')\nprint(len(train))\ntrain = pd.DataFrame([[p.split('/')[3],p.split('/')[4],p] for p in train], columns = ['type','image','path'])\n#train = train[train['size'] != '0 0'].reset_index(drop=True) #remove bad images\ntrain_data = normalize_image_features(train['path'])\n#np.save('train.npy', train_data, allow_pickle=True, fix_imports=True)\nle = LabelEncoder()\ntrain_target = le.fit_transform(train['type'].values)\n\ndef create_model(opt_):\n    model = Sequential()\n    model.add(Convolution2D(32, 3, 3, input_shape=(3, 64, 64), activation='relu', dim_ordering='th'))\n    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), dim_ordering='th'))\n    model.add(Dropout(0.2))\n    model.add(Flatten())\n    model.add(Dense(12))\n    model.add(Dropout(0.2))\n    model.add(Dense(3, activation='softmax'))\n\n    model.compile(optimizer=opt_, loss='sparse_categorical_crossentropy', metrics=['accuracy']) #loss='binary_crossentropy' not working\n    return model\n\nmodel = KerasClassifier(build_fn=create_model, nb_epoch=10, batch_size=15, verbose=2)\nopts_ = ['adamax'] #['adadelta','sgd','adagrad','adam','adamax']\nepochs = np.array([10])\nbatches = np.array([15])\nparam_grid = dict(nb_epoch=epochs, batch_size=batches, opt_=opts_)\ngrid = GridSearchCV(estimator=model, cv=StratifiedKFold(n_splits=2), param_grid=param_grid, verbose=20)\ngrid_result = grid.fit(train_data, train_target)\n\nprint(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nfor params, mean_score, scores in grid_result.grid_scores_:\n    print(\"%f (%f) with: %r\" % (scores.mean(), scores.std(), params))\n#print(\"Log Loss...\", log_loss(train_target, grid_result.predict(train_data)))\n\ntest_data = normalize_image_features(test['path'])\n#np.save('test.npy', test_data, allow_pickle=True, fix_imports=True)\ntest_id = test.image.values\n\npred = grid_result.predict_proba(test_data)\ndf = pd.DataFrame(pred, columns=le.classes_)\ndf['image_name'] = test_id\ndf.to_csv('submission.csv', index=False)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"23d35376-f29f-ce66-761e-e8f22d299cdd"},"outputs":[],"source":"df.head()"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}