{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n

SHAP https://towardsdatascience.com/explainable-ai-xai-with-shap-regression-problem-b2d63fdca670

\n\nShapley Additive Explanation -> modelleme parametrelerinin \n\n##### SHAP (SHapley Additive ExPlanations), herhangi bir ML, CNN vb. modelinin çıktısını açıklamaya yönelik birleşik bir yaklaşımdır. SHAP, oyun teorisini yerel açıklamalarla birleştirir, önceki birkaç yöntemi birleştirir ve beklentilere dayalı olası tek tutarlı ve yerel olarak doğru eklemeli özellik ilişkilendirme yöntemini temsil eder ","metadata":{}},{"cell_type":"markdown","source":"## Sabit çalışan modüller tekrardan sıfır bir sistem kurmamızdan ötürü burada tekrardan çağrılır","metadata":{}},{"cell_type":"code","source":"import os\nimport shap\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nfrom keras.models import Model\nfrom keras import optimizers, applications\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom keras.layers import Dense, Dropout, GlobalAveragePooling2D, Input\n\n# burada döngünün tekrarlanabilmesine yönelik random DR görüntüsüne ait yine random elemanlar çağrılmakta, otokorelasyondan kaçınmak amaçlı!\nfrom tensorflow import set_random_seed\ndef seed_everything(seed=0):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n set_random_seed(seed)\n\nseed = 0\nseed_everything(seed)\n# plot ayarları \n%matplotlib inline\nsns.set(style=\"whitegrid\")\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Datanın sisteme yüklenmesi bu işlem adımında gerçekleşmektedir.","metadata":{"_kg_hide-output":true}},{"cell_type":"code","source":"train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint('Number of train samples: ', train.shape[0])\nprint('Number of test samples: ', test.shape[0])\n\n# ön işlem adımları burada yürümekte.\ntrain[\"id_code\"] = train[\"id_code\"].apply(lambda x: x + \".png\")\ntest[\"id_code\"] = test[\"id_code\"].apply(lambda x: x + \".png\")\ntrain['diagnosis'] = train['diagnosis'].astype('str')\ndisplay(train.head())","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model parametreleri","metadata":{}},{"cell_type":"code","source":"# Model parametreleri\nBATCH_SIZE = 8\nEPOCHS = 40\nWARMUP_EPOCHS = 2\nLEARNING_RATE = 1e-4\nWARMUP_LEARNING_RATE = 1e-3\nHEIGHT = 320\nWIDTH = 320\nCANAL = 3\nN_CLASSES = train['diagnosis'].nunique()\nES_PATIENCE = 5\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# modülü çalışıtmak için gerekli olan test / diğer kullanımlarda gerekli değil!","metadata":{}},{"cell_type":"code","source":"X_train, X_val = train_test_split(train, test_size=0.2, random_state=seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data üretimi","metadata":{}},{"cell_type":"code","source":"train_datagen=ImageDataGenerator(rescale=1./255, \n rotation_range=360,\n horizontal_flip=True,\n vertical_flip=True)\n\ntrain_generator=train_datagen.flow_from_dataframe(\n dataframe=X_train,\n directory=\"../input/aptos2019-blindness-detection/train_images/\",\n x_col=\"id_code\",\n y_col=\"diagnosis\",\n class_mode=\"categorical\",\n batch_size=BATCH_SIZE,\n target_size=(HEIGHT, WIDTH),\n seed=0)\n\nvalidation_datagen = ImageDataGenerator(rescale=1./255)\n\nvalid_generator=validation_datagen.flow_from_dataframe(\n dataframe=X_val,\n directory=\"../input/aptos2019-blindness-detection/train_images/\",\n x_col=\"id_code\",\n y_col=\"diagnosis\",\n class_mode=\"categorical\", \n batch_size=BATCH_SIZE, \n target_size=(HEIGHT, WIDTH),\n seed=0)\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntest_generator = test_datagen.flow_from_dataframe( \n dataframe=test,\n directory = \"../input/aptos2019-blindness-detection/test_images/\",\n x_col=\"id_code\",\n batch_size=1,\n class_mode=None,\n shuffle=False,\n target_size=(HEIGHT, WIDTH),\n seed=0)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def create_model(input_shape, n_out):\n input_tensor = Input(shape=input_shape)\n base_model = applications.ResNet50(weights=None, \n include_top=False,\n input_tensor=input_tensor)\n base_model.load_weights('../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n x = GlobalAveragePooling2D()(base_model.output)\n x = Dropout(0.5)(x)\n x = Dense(2048, activation='relu')(x)\n x = Dropout(0.5)(x)\n final_output = Dense(n_out, activation='softmax', name='final_output')(x)\n model = Model(input_tensor, final_output)\n \n return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DR seviyelerinin üst tabakalarının modellenmesi ","metadata":{}},{"cell_type":"code","source":"model = create_model(input_shape=(HEIGHT, WIDTH, CANAL), n_out=N_CLASSES)\n\nfor layer in model.layers:\n layer.trainable = False\n\nfor i in range(-5, 0):\n model.layers[i].trainable = True\n \nclass_weights = class_weight.compute_class_weight('balanced', np.unique(train['diagnosis'].astype('int').values), train['diagnosis'].astype('int').values)\n\nmetric_list = [\"accuracy\"]\noptimizer = optimizers.Adam(lr=WARMUP_LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=metric_list)\nmodel.summary()","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size\nSTEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size\n\nhistory_warmup = model.fit_generator(generator=train_generator,\n steps_per_epoch=STEP_SIZE_TRAIN,\n validation_data=valid_generator,\n validation_steps=STEP_SIZE_VALID,\n epochs=WARMUP_EPOCHS,\n class_weight=class_weights,\n verbose=1).history","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Oluşturulan kaba modellere mm düzeyinde ince veri işlemin yapılması","metadata":{}},{"cell_type":"code","source":"for layer in model.layers:\n layer.trainable = True\n\nes = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\nrlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)\n\ncallback_list = [es, rlrop]\noptimizer = optimizers.Adam(lr=LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=metric_list)\nmodel.summary()","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_finetunning = model.fit_generator(generator=train_generator,\n steps_per_epoch=STEP_SIZE_TRAIN,\n validation_data=valid_generator,\n validation_steps=STEP_SIZE_VALID,\n epochs=EPOCHS,\n callbacks=callback_list,\n class_weight=class_weights,\n verbose=1).history","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model loss grafiği ","metadata":{}},{"cell_type":"code","source":"history = {'loss': history_warmup['loss'] + history_finetunning['loss'], \n 'val_loss': history_warmup['val_loss'] + history_finetunning['val_loss'], \n 'acc': history_warmup['acc'] + history_finetunning['acc'], \n 'val_acc': history_warmup['val_acc'] + history_finetunning['val_acc']}\n\nsns.set_style(\"whitegrid\")\nfig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 14))\n\nax1.plot(history['loss'], label='Train loss')\nax1.plot(history['val_loss'], label='Validation loss')\nax1.legend(loc='best')\nax1.set_title('Loss')\n\nax2.plot(history['acc'], label='Train accuracy')\nax2.plot(history['val_acc'], label='Validation accuracy')\nax2.legend(loc='best')\nax2.set_title('Accuracy')\n\nplt.xlabel('Epochs')\nsns.despine()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelin Oluşturulması\n\n## Matriks takımlarının Üretilmesi","metadata":{}},{"cell_type":"code","source":"# Modellenen tahmini değerleri ve başlıkları tutmak için boş dizilerin oluşturulması\nlastFullTrainPred = np.empty((0, N_CLASSES))\nlastFullTrainLabels = np.empty((0, N_CLASSES))\nlastFullValPred = np.empty((0, N_CLASSES))\nlastFullValLabels = np.empty((0, N_CLASSES))\n\nfor i in range(STEP_SIZE_TRAIN+1):\n im, lbl = next(train_generator)\n scores = model.predict(im, batch_size=train_generator.batch_size)\n lastFullTrainPred = np.append(lastFullTrainPred, scores, axis=0)\n lastFullTrainLabels = np.append(lastFullTrainLabels, lbl, axis=0)\n\n# örnekleme aralığı ve başlıkların otomatik olarak çekilmesi \nfor i in range(STEP_SIZE_VALID+1):\n im, lbl = next(valid_generator)\n scores = model.predict(im, batch_size=valid_generator.batch_size)\n lastFullValPred = np.append(lastFullValPred, scores, axis=0)\n lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)\n \n \nlastFullComPred = np.concatenate((lastFullTrainPred, lastFullValPred))\nlastFullComLabels = np.concatenate((lastFullTrainLabels, lastFullValLabels))\ncomplete_labels = [np.argmax(label) for label in lastFullComLabels]\n\ntrain_preds = [np.argmax(pred) for pred in lastFullTrainPred]\ntrain_labels = [np.argmax(label) for label in lastFullTrainLabels]\nvalidation_preds = [np.argmax(pred) for pred in lastFullValPred]\nvalidation_labels = [np.argmax(label) for label in lastFullValLabels]","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, sharex='col', figsize=(24, 7))\nlabels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ntrain_cnf_matrix = confusion_matrix(train_labels, train_preds)\nvalidation_cnf_matrix = confusion_matrix(validation_labels, validation_preds)\n\ntrain_cnf_matrix_norm = train_cnf_matrix.astype('float') / train_cnf_matrix.sum(axis=1)[:, np.newaxis]\nvalidation_cnf_matrix_norm = validation_cnf_matrix.astype('float') / validation_cnf_matrix.sum(axis=1)[:, np.newaxis]\n\ntrain_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)\nvalidation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)\n\nsns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap=\"Blues\", ax=ax1).set_title('Train')\nsns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8), ax=ax2).set_title('Validation')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## İkinci Derece Ağırlıklandımra ****matriks takımı içerisinde LSQ şeklinde Kappa ağırlıklandırması yapılmakta, buradan elde edilen skor değeri modellemenin başarı oranını simgelemekte","metadata":{}},{"cell_type":"code","source":"print(\"Train Cohen Kappa score: %.3f\" % cohen_kappa_score(train_preds,train_labels, weights='quadratic'))\nprint(\"Validation Cohen Kappa score: %.3f\" % cohen_kappa_score(validation_preds, validation_labels, weights='quadratic'))\nprint(\"Complete set Cohen Kappa score: %.3f\" % cohen_kappa_score(train_preds+validation_preds, train_labels+validation_labels, weights='quadratic'))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SHAP Modellerin çizdirimi ve yorumlanması","metadata":{}},{"cell_type":"code","source":"n_explain = 2\nvalid_generator.batch_size = 10 # tezin ilk kısmından elde edilen verisetinin çizdirilmesi\nbackground, lbls = next(valid_generator)\n\nsns.set_style(\"white\")\nplt.figure(figsize=[8, 8])\nfor index, image in enumerate(background[:n_explain]):\n plt.subplot(n_explain, 1, index+1)\n plt.imshow(image)\n plt.title(\"Image %s, Label: %s\" % (index, np.argmax(lbls[index])))\n \nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ve SHAP ile bu değerlerin eldesi","metadata":{}},{"cell_type":"code","source":"\ne = shap.DeepExplainer(model, background)\nshap_values = e.shap_values(background)\n\n\nshap.image_plot(shap_values, -background[:n_explain], labels=lbls, hspace=0.1)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Yukarıdaki çizim, üç farklı görüntü için beş çıktıyı (bizim beş diyabetik retinopati 0-5 seviyemiz) açıklamaktadır. Kırmızı pikseller modelin çıktısını artırırken mavi pikseller çıktıyı azaltır. Giriş görüntüleri solda (piksellerin çoğu 0'dan büyük olduğundan siyahtır) ve açıklamaların her birinin arkasında neredeyse saydam gri tonlamalı destekler olarak gösterilir. SHAP değerlerinin toplamı, beklenen model çıktısı (arka plan veri kümesi üzerinden ortalaması alınır, burada 10 görüntü kullanıyoruz) ile mevcut model çıktısı arasındaki farka eşittir.\n- başlığın \"1.0\" olduğu resimler için daha büyük bir pembe alanımız olduğunu unutmayın.\n- Doğru olan kadar pembe alana sahip etiketler, modelimizin muhtemelen yüksek bir güven tahminine sahip olmadığı kısımları ifade etmektedir, tez sunumunda muhtemel sorulacak bir soru!","metadata":{}},{"cell_type":"markdown","source":"## diğer örnekler","metadata":{}},{"cell_type":"code","source":"n_explain = 3\nbackground, lbls = next(valid_generator)\n\nsns.set_style(\"white\")\nplt.figure(figsize=[12, 12])\nfor index, image in enumerate(background[:n_explain]):\n plt.subplot(n_explain, 1, index+1)\n plt.imshow(image)\n plt.title(\"Image %s, Label: %s\" % (index, np.argmax(lbls[index])))\n \nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ne = shap.DeepExplainer(model, background)\nshap_values = e.shap_values(background)\n\n\nshap.image_plot(shap_values, -background[:n_explain], labels=lbls, hspace=0.1)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeli test seti ve çıktı tahminlerine uygulanması","metadata":{}},{"cell_type":"code","source":"test_generator.reset()\nSTEP_SIZE_TEST = test_generator.n//test_generator.batch_size\npreds = model.predict_generator(test_generator, steps=STEP_SIZE_TEST)\npredictions = [np.argmax(pred) for pred in preds]\n\nfilenames = test_generator.filenames\nresults = pd.DataFrame({'id_code':filenames, 'diagnosis':predictions})\nresults['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tahminler dağılımı","metadata":{}},{"cell_type":"code","source":"fig = plt.subplots(1, 1, sharex='col', figsize=(24, 8.7))\nsns.countplot(x=\"diagnosis\", data=results, palette=\"GnBu_d\")\nsns.despine()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.to_csv('submission.csv', index=False)\nresults.head(10)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}