{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom albumentations import Compose, Flip, CropAndPad, Transpose\nimport glob, os\nimport torch\nimport torchvision\nfrom skimage import io\nfrom fastprogress import master_bar, progress_bar\n\ntorch.__version__\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-29T15:53:48.931144Z","iopub.execute_input":"2024-02-29T15:53:48.931609Z","iopub.status.idle":"2024-02-29T15:53:48.941050Z","shell.execute_reply.started":"2024-02-29T15:53:48.931568Z","shell.execute_reply":"2024-02-29T15:53:48.940157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the train\ntrain = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:48.942755Z","iopub.execute_input":"2024-02-29T15:53:48.943270Z","iopub.status.idle":"2024-02-29T15:53:49.041789Z","shell.execute_reply.started":"2024-02-29T15:53:48.943240Z","shell.execute_reply":"2024-02-29T15:53:49.040624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the test\ntest = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:49.044260Z","iopub.execute_input":"2024-02-29T15:53:49.044871Z","iopub.status.idle":"2024-02-29T15:53:49.082056Z","shell.execute_reply.started":"2024-02-29T15:53:49.044827Z","shell.execute_reply":"2024-02-29T15:53:49.081177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['target'].hist()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:49.083428Z","iopub.execute_input":"2024-02-29T15:53:49.084008Z","iopub.status.idle":"2024-02-29T15:53:49.296431Z","shell.execute_reply.started":"2024-02-29T15:53:49.083976Z","shell.execute_reply":"2024-02-29T15:53:49.294764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['sex'].hist()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:49.299818Z","iopub.execute_input":"2024-02-29T15:53:49.300138Z","iopub.status.idle":"2024-02-29T15:53:49.492368Z","shell.execute_reply.started":"2024-02-29T15:53:49.300111Z","shell.execute_reply":"2024-02-29T15:53:49.491349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['anatom_site_general_challenge'].hist()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:49.496316Z","iopub.execute_input":"2024-02-29T15:53:49.496618Z","iopub.status.idle":"2024-02-29T15:53:49.708377Z","shell.execute_reply.started":"2024-02-29T15:53:49.496591Z","shell.execute_reply":"2024-02-29T15:53:49.706977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['age_approx'].hist()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:49.709897Z","iopub.execute_input":"2024-02-29T15:53:49.710298Z","iopub.status.idle":"2024-02-29T15:53:49.884498Z","shell.execute_reply.started":"2024-02-29T15:53:49.710268Z","shell.execute_reply":"2024-02-29T15:53:49.883384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train['path'] = [f'/kaggle/input/siim-isic-melanoma-classification/jpeg/train/{img}.jpg' for img in train['image_name']]\nimport glob\n\nimport os\n# assign directory\ndirectory_train = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\n \n# iterate over files in\n# that directory\nfor filename in os.listdir(directory_train)[0:20]: \n    f = os.path.join(directory_train, filename)\n\n    #print(f)\n    im = plt.imread(f)\n    \n    plt.figure(figsize=(5,10))\n    plt.imshow(im)\n    plt.axis('off')\n    plt.show()\n    \n\n#for file in glob.glob(f'/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'):\n#    print (file)\n    #im = plt.imread(\"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/\" file)\n    #plt.figure(figsize=(15,20))\n    #plt.imshow(im)\n    #plt.show()\n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:49.886180Z","iopub.execute_input":"2024-02-29T15:53:49.886880Z","iopub.status.idle":"2024-02-29T15:53:53.521331Z","shell.execute_reply.started":"2024-02-29T15:53:49.886841Z","shell.execute_reply":"2024-02-29T15:53:53.519470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom as dicom\nimport matplotlib.pylab as plt\nimport cv2   \nimport matplotlib.pyplot as plt\n\n\n# specify your image path\n#directory_test = '/kaggle/input/siim-isic-melanoma-classification/test/'\n#ds = dicom.dcmread(directory_test)#\n\n#plt.imshow(ds.pixel_array)\n\n# convert dcm to jpg\ndirectory_test = '/kaggle/input/siim-isic-melanoma-classification/test/ISIC_0052060.dcm'\nds = dicom.dcmread(directory_test)\n\npixel_array_numpy = ds.pixel_array\n\nimage_format = '.jpg' # or '.png'\ndirectory_test = directory_test.replace('.dcm', image_format)\n\nplt.imshow(pixel_array_numpy)\n\ncv2.imwrite(directory_test, pixel_array_numpy)","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:53.522289Z","iopub.status.idle":"2024-02-29T15:53:53.522674Z","shell.execute_reply.started":"2024-02-29T15:53:53.522472Z","shell.execute_reply":"2024-02-29T15:53:53.522487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pydicom.pixel_data_handlers import convert_color_space\n\nrgb = convert_color_space(pixel_array_numpy , \"YBR_FULL\", \"RGB\")\nplt.imshow(rgb)","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:53.524819Z","iopub.status.idle":"2024-02-29T15:53:53.525370Z","shell.execute_reply.started":"2024-02-29T15:53:53.525076Z","shell.execute_reply":"2024-02-29T15:53:53.525101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import csv \n\n#CHALLENGE 2\n\ninfile = '/kaggle/input/siim-isic-melanoma-classification/train.csv'\noutfile = 'ch2.csv'\nn1=0\nn2=0\n\nwith open(infile, encoding='utf-8') as f, open(outfile, 'w') as o:\n    reader = csv.reader(f)\n    writer = csv.writer(o, delimiter=',') # adjust as necessary\n    for row in reader:\n       if row[7] == \"1\" and n1 < 100:\n            writer.writerow(row)\n            n1 = n1 + 1\n       if row[7] == \"0\" and n2 < 100:\n            writer.writerow(row)\n            n2 = n2 +1","metadata":{"execution":{"iopub.status.busy":"2024-02-29T17:02:41.642728Z","iopub.execute_input":"2024-02-29T17:02:41.643275Z","iopub.status.idle":"2024-02-29T17:02:41.722884Z","shell.execute_reply.started":"2024-02-29T17:02:41.643240Z","shell.execute_reply":"2024-02-29T17:02:41.721366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import layers, callbacks\nfrom keras.losses import BinaryCrossentropy\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\nfrom keras import backend as K\nimport tensorflow as tf\nimport csv\n\n\nfile = 'ch2.csv'\n#total_img = ch2.csv['target'].size\n\nreader = csv.reader(open(\"ch2.csv\"))\ntotal_img= len(list(reader))\n\nmalignant = 100\nbenign = total_img - malignant\n\nprint('Examples:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n    total_img, malignant, 100 * malignant / total_img))","metadata":{"execution":{"iopub.status.busy":"2024-02-29T15:53:53.529380Z","iopub.status.idle":"2024-02-29T15:53:53.530067Z","shell.execute_reply.started":"2024-02-29T15:53:53.529871Z","shell.execute_reply":"2024-02-29T15:53:53.529889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications.vgg16 import VGG16\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications.vgg16 import preprocess_input\nfrom sklearn.linear_model import LinearRegression\nimport numpy as np\n\n\nmodel= VGG16(weights='imagenet', include_top=False)\n\ninfile = 'ch2.csv'\ninput_folder = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\nfeature_list= []\ntipologia = []\nwith open(infile, encoding='utf-8') as f:\n    reader = csv.reader(f)\n    for row in reader:\n        \n       #print (input_folder+row[0]+'.jpg', row[7])\n       img = image.load_img(input_folder+row[0]+'.jpg', target_size=(224, 224))\n       x= image.img_to_array(img)\n       x= np.expand_dims(x, axis=0)\n       x= preprocess_input(x)\n       features = model.predict(x)\n       feature_list.append(features);\n    \n       if row[7] == \"0\":\n           tipus=0\n       else: \n           tipus=1\n       tipologia.append(tipus);\n       #print (features)\n        \n        \n    #feature_list = np.array(feature_list) \n    #tipologia = np.array(tipologia)\n    \nfeature_list_rs= np.reshape(feature_list,(-1,7*7*512))\n    \n    #reg = LinearRegression().fit(feature_list, tipologia)\n    #reg.score(feature_list, tipologia)\n\n    \n        \n        \n#print(features.shape)\n#print(features)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-29T16:32:35.371176Z","iopub.execute_input":"2024-02-29T16:32:35.371626Z","iopub.status.idle":"2024-02-29T16:33:55.469326Z","shell.execute_reply.started":"2024-02-29T16:32:35.371594Z","shell.execute_reply":"2024-02-29T16:33:55.468233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(type(feature_list))\nprint(feature_list)\n#print(tipologia)\n#reg = LinearRegression().fit(feature_list, tipologia)\n    #reg.score(feature_list, tipologia)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-29T16:03:30.618867Z","iopub.execute_input":"2024-02-29T16:03:30.619347Z","iopub.status.idle":"2024-02-29T16:03:30.641583Z","shell.execute_reply.started":"2024-02-29T16:03:30.619314Z","shell.execute_reply":"2024-02-29T16:03:30.640421Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nclf = LogisticRegression(random_state=0).fit(feature_list_rs, tipologia)\nscore= clf.score(feature_list_rs,  tipologia)\nprint(score)","metadata":{"execution":{"iopub.status.busy":"2024-02-29T16:38:14.601179Z","iopub.execute_input":"2024-02-29T16:38:14.601577Z","iopub.status.idle":"2024-02-29T16:38:15.919476Z","shell.execute_reply.started":"2024-02-29T16:38:14.601538Z","shell.execute_reply":"2024-02-29T16:38:15.918172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\npred= clf.predict(feature_list_rs)\n\nmat= confusion_matrix(tipologia,pred)\nprint(mat)","metadata":{"execution":{"iopub.status.busy":"2024-02-29T16:52:46.752621Z","iopub.execute_input":"2024-02-29T16:52:46.753635Z","iopub.status.idle":"2024-02-29T16:52:46.786275Z","shell.execute_reply.started":"2024-02-29T16:52:46.753593Z","shell.execute_reply":"2024-02-29T16:52:46.784605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#per treure overfiting\ninfile = '/kaggle/input/siim-isic-melanoma-classification/train.csv'\noutfile = 'ch3.csv'\nn1=0\nn2=0\n\nwith open(infile, encoding='utf-8') as f, open(outfile, 'w') as o:\n    reader = csv.reader(f)\n    writer = csv.writer(o, delimiter=',') # adjust as necessary\n    for row in reader:\n       \n       if row[7] == \"1\" and n1 <= 100:\n            n1 = n1 + 1\n       elif row[7] == \"1\" and n1 > 100 and n1 < 200:\n            writer.writerow(row)\n            n1 = n1 + 1\n       #if row[7] == \"0\" and n2 > 100:\n       #     writer.writerow(row)\n       #     n2 = n2 + 1","metadata":{"execution":{"iopub.status.busy":"2024-02-29T17:13:29.476335Z","iopub.execute_input":"2024-02-29T17:13:29.476858Z","iopub.status.idle":"2024-02-29T17:13:29.557226Z","shell.execute_reply.started":"2024-02-29T17:13:29.476822Z","shell.execute_reply":"2024-02-29T17:13:29.555815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = 'ch2.csv'\n#total_img = ch2.csv['target'].size\n\nreader = csv.reader(open(\"ch2.csv\"))\ntotal_img= len(list(reader))\n\nmalignant = 100\nbenign = total_img - malignant\n\nprint('Examples:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n    total_img, malignant, 100 * malignant / total_img))","metadata":{"execution":{"iopub.status.busy":"2024-02-29T17:01:35.655550Z","iopub.execute_input":"2024-02-29T17:01:35.656060Z","iopub.status.idle":"2024-02-29T17:01:35.714933Z","shell.execute_reply.started":"2024-02-29T17:01:35.656023Z","shell.execute_reply":"2024-02-29T17:01:35.712923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nclf = LogisticRegression(random_state=0).fit(feature_list_rs, tipologia)\nscore= clf.score(feature_list_rs,  tipologia)\nprint(score)","metadata":{"execution":{"iopub.status.busy":"2024-02-29T16:59:54.767561Z","iopub.execute_input":"2024-02-29T16:59:54.769067Z","iopub.status.idle":"2024-02-29T16:59:56.230221Z","shell.execute_reply.started":"2024-02-29T16:59:54.769013Z","shell.execute_reply":"2024-02-29T16:59:56.228601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\npred= clf.predict(feature_list_rs)\n\nmat= confusion_matrix(tipologia,pred)\nprint(mat)","metadata":{},"execution_count":null,"outputs":[]}]}