{"metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "nbconvert_exporter": "python", "mimetype": "text/x-python", "version": "3.6.3", "pygments_lexer": "ipython3", "file_extension": ".py", "name": "python"}}, "nbformat": 4, "nbformat_minor": 1, "cells": [{"metadata": {"_uuid": "14d5c7e1788c59c283fd0b0e04615d07ebe8b801", "_cell_guid": "5fb10f1d-4ae2-42c5-89b3-acc6718f29ea", "collapsed": true}, "source": ["# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "#from subprocess import check_output\n", "#print(check_output([\"ls\", \"../input/batchtestbis/fBATCH\"]).decode(\"utf8\"))\n"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "bbbf536a01b6d584698d74eaddc59dcee539b311", "_cell_guid": "d4ae8c10-9747-4661-9c0a-0fc3b9e76ce5"}, "source": ["# Passenger Screening Challenge Kaggle"], "cell_type": "markdown"}, {"metadata": {"_uuid": "311cd0b8601d86ec2c93c3c9583d87ceb28eecf3", "_cell_guid": "acbe6949-2070-41eb-b26c-d151ed0994ab"}, "source": ["## Summary"], "cell_type": "markdown"}, {"metadata": {"_uuid": "3e6d846a43702f31c874edef07495c5c0459928c", "_cell_guid": "1dec7050-fb0a-42e6-b2c1-0febcef2a95f"}, "source": ["## Introduction\n", "In this competition, TSA is stepping outside their established procurement process and is challenging the broader data science community to help improve the accuracy of their threat prediction algorithms. Using a dataset of images collected on the latest generation of scanners, participants are challenged to identify the presence of simulated threats under a variety of object types, clothing types, and body types. Even a modest decrease in false alarms will help TSA significantly improve the passenger experience while maintaining high levels of security.\n", "\n", "    _.aps = projected image angle sequence file (10.3MB per file)\n", "    _.a3d = combined image 3D file (330MB per file)\n", "    _.a3daps = combined image angle sequence file (41.2MB per file)\n", "    _.ahi = calibrated object raw data file (2.26GB per file)\n", "    \n", "The data type of this file is 16bit unsigned integer. Data scaling is achieved by multiplying each pixel value by the 'data_scale_factor' field in the header. "], "cell_type": "markdown"}, {"metadata": {"_uuid": "274da5a35ca5828f83959c052824c008a3e3adea", "_cell_guid": "ec8d5b07-b156-4e54-96fd-3fb1a8562803"}, "source": ["## Libraries"], "cell_type": "markdown"}, {"metadata": {"_uuid": "2657e7ae50c605103c099d10e7f7753f34b7fe40", "_cell_guid": "b096ace7-85bd-410f-b26d-33627e938fdc", "collapsed": true}, "source": ["import pandas as pd #used for the data frame manipulation\n", "import numpy as np #used for the array manipulation\n", "import os #used for the interface operating system\n", "\n", "import sklearn as skl #used for machine learning algorithm\n", "from sklearn.cluster import MeanShift, estimate_bandwidth\n", "\n", "import matplotlib #used for the visualisation\n", "import matplotlib.pyplot as plt #used for the style of Matlab function\n", "import matplotlib.animation as anim # used for life animation of Matplotlib\n", "\n", "%matplotlib inline"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "4e8a30cb79fd0fd3ad37bebf8503bd392a9ced7c", "_cell_guid": "7bf4b784-36de-4294-8cc1-f03ddd434049", "collapsed": true}, "source": ["os.sys.path"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "0f63dcd4059e9e2462ec92d33428eb6d8f2d664b", "_cell_guid": "b0e63e96-4eb7-4eca-9125-1290c5d98a8c", "collapsed": true}, "source": ["import platform\n", "platform.architecture()"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "b856f0b26a4b23ffa0fe10c5e6cb51b715221cc3", "_cell_guid": "2c02fe01-3dfd-4abd-bec8-28ed10ed615c", "collapsed": true}, "source": ["#import cv2"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "18479d65f29ef57615e9ba735d3c14f73d983ccf", "_cell_guid": "417f7f05-eae5-43ed-b12f-85063652ad04"}, "source": ["## Macro Variables"], "cell_type": "markdown"}, {"metadata": {"_uuid": "f08bb34f5409cf8ed7472b8344866cac0afdc48c", "_cell_guid": "bd36254b-05b8-4574-808c-bf2dfc9c8d1d", "collapsed": true}, "source": ["pathData = os.path.join(\"../input\")\n", "dataTrain = os.path.join(pathData,\"passenger-screening-algorithm-challenge/stage1_labels.csv\")\n", "dataSub = os.path.join(pathData,\"passenger-screening-algorithm-challenge/stage1_sample_submission.csv\")\n", "dataAps= os.path.join(pathData,\"batchtest\")\n", "dataApsBis= os.path.join(pathData,\"batchtestbis\")"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "164811edd52da71dd24cca9ac142aab8ae024680", "_cell_guid": "cdc0d1a6-5642-417d-b34f-5355ce9631f2"}, "source": ["## Macro functions"], "cell_type": "markdown"}, {"metadata": {"_uuid": "49a706aecb4dd3882f4c106e4f529854155e7223", "_cell_guid": "1e72ad05-c895-494a-a88d-f1ab9c2c0fbe", "collapsed": true}, "source": ["# To read the header of the files\n", "def read_header(infile):\n", "    \"\"\"Read image header (first 512 bytes)\n", "    \"\"\"\n", "    h = dict()\n", "    fid = open(infile, 'rb')#+b\n", "    h['filename'] = b''.join(np.fromfile(fid, dtype = 'S1', count = 20))\n", "    h['parent_filename'] = b''.join(np.fromfile(fid, dtype = 'S1', count = 20))\n", "    h['comments1'] = b''.join(np.fromfile(fid, dtype = 'S1', count = 80))\n", "    h['comments2'] = b''.join(np.fromfile(fid, dtype = 'S1', count = 80))\n", "    h['energy_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['config_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['file_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['trans_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['scan_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['data_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['date_modified'] = b''.join(np.fromfile(fid, dtype = 'S1', count = 16))\n", "    h['frequency'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['mat_velocity'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['num_pts'] = np.fromfile(fid, dtype = np.int32, count = 1)\n", "    h['num_polarization_channels'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['spare00'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['adc_min_voltage'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['adc_max_voltage'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['band_width'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['spare01'] = np.fromfile(fid, dtype = np.int16, count = 5)\n", "    h['polarization_type'] = np.fromfile(fid, dtype = np.int16, count = 4)\n", "    h['record_header_size'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['word_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['word_precision'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['min_data_value'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['max_data_value'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['avg_data_value'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['data_scale_factor'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['data_units'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['surf_removal'] = np.fromfile(fid, dtype = np.uint16, count = 1)\n", "    h['edge_weighting'] = np.fromfile(fid, dtype = np.uint16, count = 1)\n", "    h['x_units'] = np.fromfile(fid, dtype = np.uint16, count = 1)\n", "    h['y_units'] = np.fromfile(fid, dtype = np.uint16, count = 1)\n", "    h['z_units'] = np.fromfile(fid, dtype = np.uint16, count = 1)\n", "    h['t_units'] = np.fromfile(fid, dtype = np.uint16, count = 1)\n", "    h['spare02'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['x_return_speed'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['y_return_speed'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['z_return_speed'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['scan_orientation'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['scan_direction'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['data_storage_order'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['scanner_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['x_inc'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['y_inc'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['z_inc'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['t_inc'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['num_x_pts'] = np.fromfile(fid, dtype = np.int32, count = 1)\n", "    h['num_y_pts'] = np.fromfile(fid, dtype = np.int32, count = 1)\n", "    h['num_z_pts'] = np.fromfile(fid, dtype = np.int32, count = 1)\n", "    h['num_t_pts'] = np.fromfile(fid, dtype = np.int32, count = 1)\n", "    h['x_speed'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['y_speed'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['z_speed'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['x_acc'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['y_acc'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['z_acc'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['x_motor_res'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['y_motor_res'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['z_motor_res'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['x_encoder_res'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['y_encoder_res'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['z_encoder_res'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['date_processed'] = b''.join(np.fromfile(fid, dtype = 'S1', count = 8))\n", "    h['time_processed'] = b''.join(np.fromfile(fid, dtype = 'S1', count = 8))\n", "    h['depth_recon'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['x_max_travel'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['y_max_travel'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['elevation_offset_angle'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['roll_offset_angle'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['z_max_travel'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['azimuth_offset_angle'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['adc_type'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['spare06'] = np.fromfile(fid, dtype = np.int16, count = 1)\n", "    h['scanner_radius'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['x_offset'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['y_offset'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['z_offset'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['t_delay'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['range_gate_start'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['range_gate_end'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['ahis_software_version'] = np.fromfile(fid, dtype = np.float32, count = 1)\n", "    h['spare_end'] = np.fromfile(fid, dtype = np.float32, count = 10)\n", "    return h"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "e79dcecdb8207d1f1a865b113ae01b7b16a6d1ad", "_cell_guid": "7d0d2539-64c9-403c-bb02-5da267bd3b0b", "collapsed": true}, "source": ["# To read the data of the files\n", "def read_data(infile):\n", "    \"\"\"Read any of the 4 types of image files, returns a numpy array of the image contents\n", "    \"\"\"\n", "    extension = os.path.splitext(infile)[1]\n", "    h = read_header(infile)\n", "    nx = int(h['num_x_pts'])\n", "    ny = int(h['num_y_pts'])\n", "    nt = int(h['num_t_pts'])\n", "    fid = open(infile, 'rb')\n", "    fid.seek(512) #skip header\n", "    if extension == '.aps' or extension == '.a3daps':\n", "        if(h['word_type']==7): #float32\n", "            data = np.fromfile(fid, dtype = np.float32, count = nx * ny * nt)\n", "        elif(h['word_type']==4): #uint16\n", "            data = np.fromfile(fid, dtype = np.uint16, count = nx * ny * nt)\n", "        data = data * h['data_scale_factor'] #scaling factor\n", "        data = data.reshape(nx, ny, nt, order='F').copy() #make N-d image\n", "    elif extension == '.a3d':\n", "        if(h['word_type']==7): #float32\n", "            data = np.fromfile(fid, dtype = np.float32, count = nx * ny * nt)\n", "        elif(h['word_type']==4): #uint16\n", "            data = np.fromfile(fid, dtype = np.uint16, count = nx * ny * nt)\n", "        data = data * h['data_scale_factor'] #scaling factor\n", "        data = data.reshape(nx, nt, ny, order='F').copy() #make N-d image\n", "    elif extension == '.ahi':\n", "        data = np.fromfile(fid, dtype = np.float32, count = 2* nx * ny * nt)\n", "        data = data.reshape(2, ny, nx, nt, order='F').copy()\n", "        real = data[0,:,:,:].copy()\n", "        imag = data[1,:,:,:].copy()\n", "    fid.close()\n", "    if extension != '.ahi':\n", "        return data\n", "    else:\n", "        return real, imag"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "47718710bbf8a0d461fcac40c691cf8426a8fd52", "_cell_guid": "d4ea2c83-e02b-46cc-b0ca-60c1157b0f03", "collapsed": true}, "source": ["# To build the animation of Matplotlib\n", "matplotlib.rc('animation', html='html5')\n", "\n", "def plot_image(path):\n", "    data = read_data(path)\n", "    fig = matplotlib.pyplot.figure(figsize = (16,16))\n", "    ax = fig.add_subplot(111)\n", "    def animate(i):\n", "        im = ax.imshow(np.flipud(data[:,:,i].transpose()), cmap = 'viridis')\n", "        return [im]\n", "    return matplotlib.animation.FuncAnimation(fig, animate, frames=range(0,data.shape[2]), interval=200, blit=True)"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "898e220ca889779e690d70e77063d5468ccd3287", "_cell_guid": "1de39bd0-ef30-4120-953d-5ee858e38660", "collapsed": true}, "source": ["## Import Data"], "cell_type": "markdown"}, {"metadata": {"_uuid": "94ebea6b36325236dd679d5d9e5f8467e1450e41", "_cell_guid": "e1468d8b-7ee9-4c98-91b4-d8aa905d4bde", "collapsed": true}, "source": ["stgdf= pd.read_csv(dataTrain)\n", "stgdf.head()"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "6e1962532bd3cdd381e59a41f0b5d30c7d452459", "_cell_guid": "7279d4a0-3896-4769-8002-92fa52494e70"}, "source": ["\n", "## Transform Data"], "cell_type": "markdown"}, {"metadata": {"_uuid": "e775e3b97349e528716789b617d8d2981f0659a6", "_cell_guid": "ce1e3f06-2568-4d3a-8827-530a99307fd5", "collapsed": true}, "source": ["stgdf['Ind'], stgdf['Zone'] = stgdf['Id'].str.split('_Zone', 1).str\n", "stgdf['Zone']=stgdf['Zone'].apply(pd.to_numeric)\n", "stgdf.head()"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "23d9c4c3ac79f960a38e79c4e7f192a75f3b0446", "_cell_guid": "0306019b-3a68-40d8-af92-cce2fe9603a3"}, "source": ["## Explore Data"], "cell_type": "markdown"}, {"metadata": {"_uuid": "6f22bc6bac53dd281408da4abab6fdb9ab974d49", "_cell_guid": "8fabc28d-8b5a-4ca2-9d6b-8019d751f6ec"}, "source": ["81% of the individuals are reported at least with one dangerous object. It represents for 928 individuals out of 1147."], "cell_type": "markdown"}, {"metadata": {"_uuid": "34b281180a70b49d93a1e02b1c764d4a643829c0", "_cell_guid": "63927ffd-eafb-48b5-a388-4f1cf40a0901", "collapsed": true}, "source": ["df=stgdf.groupby('Ind')[['Probability']].max()\n", "df['Probability'].describe()\n", "df['Probability'].value_counts()"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "b20c08b3f0990929e8e55eb16d83cd42ac48e5b6", "_cell_guid": "e9f0a8e3-cca8-495a-b3bf-815196a7ce73"}, "source": ["Let look now which zone is holding more dangerous object"], "cell_type": "markdown"}, {"metadata": {"_uuid": "a3647c1dfc6dce1a424c8a3f1649d848afc73ecc", "_cell_guid": "6938fc24-f489-4091-8b53-d904a3581b34", "collapsed": true}, "source": ["df=stgdf.groupby('Zone',as_index=False)[['Probability']].sum()\n", "df.sort_values(['Probability'],ascending=False)\n", "#plt.bar(df.Zone, df.Probability, align='center', alpha=0.5)\n", "#plt.ylabel('Usage')\n", "#plt.title('Programming language usage')"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "6c5bd6a43a39b1697f358caf93849fdeba3a589f", "_cell_guid": "274e885c-b531-43bb-9473-16dfd505c143"}, "source": ["## Open aps files\n", "\n"], "cell_type": "markdown"}, {"metadata": {"_uuid": "ce4286eaef61275d3fef55207c03bc939a678e03", "_cell_guid": "3e2ac6e7-07a3-40f7-8159-6eba6ec1bbea"}, "source": ["### List aps file names"], "cell_type": "markdown"}, {"metadata": {"_uuid": "e540e4f78dd61282dcdd1f7e30e50939fd80d15a", "scrolled": true, "_cell_guid": "ec957290-2d0b-42c3-8f0c-d5dddd2dc4ce", "collapsed": true}, "source": ["names_file=stgdf['Ind'].unique()\n", "names_file=names_file+'.aps'\n", "folder_names_file=[str(x[0])+'BATCH/' for x in names_file]\n", "folder_names_file=[m+n for m,n in zip(folder_names_file,names_file)]\n", "len(names_file)\n", "import random\n", "c = list(zip(names_file, folder_names_file))\n", "random.shuffle(c)\n", "names_file, folder_names_file = zip(*c)\n", "nind=50\n", "ratio=0.9"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "50c3b8557d6835f8d72198ff3ed39f27a9b53718", "scrolled": true, "_cell_guid": "385f2e7d-8aec-4dd5-9159-2bada6063c52", "collapsed": true}, "source": ["for zip_file in c[:nind]:\n", "    #names_file, folder_names_file=zip(*zip_file)\n", "    print(zip_file[0])\n", "    print(zip_file[1])"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "c1eb596eab7708785cef05b698cd13a9eea66198", "_cell_guid": "bd2d8e02-8e1d-4323-bee6-036f12c17d38", "collapsed": true}, "source": ["names_file=names_file[:nind]\n", "folder_names_file=folder_names_file[:nind]"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "5c0cbe06e240c08c1bd95ac38d96bc0bc8a3dba9", "_cell_guid": "6e66c26c-044c-47b7-9f4f-35a3ce01e08c"}, "source": ["### Limited List aps file of nind random persons"], "cell_type": "markdown"}, {"metadata": {"_uuid": "453dcad33dda640adab5765decd20b4d64c11a52", "_cell_guid": "abcbfce8-97c7-44b4-aaa3-f58e781784d7", "collapsed": true}, "source": ["data_file = {zip_file[0] : read_data(os.path.join(dataApsBis,zip_file[1])) if zip_file[0][0]=='f' else read_data(os.path.join(dataAps,zip_file[1])) for zip_file in c[:nind]}"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "6fe0684425ae81764746db45af39aa74659d6e72", "_cell_guid": "149d3f3f-2aa9-4689-bf03-dd23d5de9825"}, "source": ["### Animation of the .aps file"], "cell_type": "markdown"}, {"metadata": {"_uuid": "0772983977bed1e08d83dedecfeb4cd9d9a8d2b1", "_cell_guid": "7c062f25-4283-4951-8d09-6dcf14155084"}, "source": ["Regarding the nature of the image of type _.aps\n", "\n", "    The 'Projected Image' algorithm computes 3D images for 90-degree segments of data that are equally spaced around the region scanned. A maximum value projection of the result of each of these computations is written sequentially into a single file. The result of this is an image file that, when played back plane-by-plane, appears like the object is spinning on the screen.\n", "\n", "    Data file order: AYX (angle, vertical axis, horizontal axis)\n", "    Axis Name, Stride, Number of samples, Axis Length\n", "    XAxis, 1, Nx=512, Lx=1.0 meters\n", "    YAxis, 512, Ny=660, Ly=2.0955 meters\n", "    Angular, 337920, Na=16, La=360-degrees\n", "\n", "    The data type of this file is 16bit unsigned integer. Data scaling is achieved by multiplying each pixel value by the 'data_scale_factor' field in the header. In summary, the 'Projected Image Angle Sequence' file is a series of 2D mmWave snapshots equally spaced in angle around the object."], "cell_type": "markdown"}, {"metadata": {"_uuid": "41ce610d13b3bd204b4caf237faa0b7c8735572f", "scrolled": true, "_cell_guid": "e8381e7b-6e91-430a-9040-83b7acb62480", "collapsed": true}, "source": ["k=0\n", "data_file1=data_file[c[k][0]]\n", "if c[k][0][0]=='f' :\n", "    read_header(os.path.join(dataApsBis,c[k][1]))\n", "else :\n", "    read_header(os.path.join(dataAps,c[k][1]))\n", "    "], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "1ba63c22a96e2a6631ed780f33380ff9c12587a7", "scrolled": true, "_cell_guid": "b28eb230-2620-4ca1-a08c-79f0e1e27fbe", "collapsed": true}, "source": ["a=np.asarray(stgdf[stgdf.Ind==names_file[k].replace('.aps', '')].sort_values(by=['Zone'])['Probability'])\n", "b=stgdf[stgdf.Ind==names_file[k].replace('.aps', '')].sort_values(by=['Zone'])\n", "a"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "b6f362b80777dbf39cbcaa49a3cb1643b6b66e7c", "_cell_guid": "a8c961c5-90cc-44a4-80ca-5a4b0fc9e017", "collapsed": true}, "source": ["#Based on the tflearn example located here:\n", "#https://github.com/tflearn/tflearn/blob/master/examples/images/convnet_cifar10.py\n", "\n", "from __future__ import division, print_function, absolute_import\n", "\n", "# Import tflearn and some helpers\n", "import tensorflow as tf\n", "import tflearn\n", "from tflearn.data_utils import shuffle\n", "from tflearn.layers.core import input_data, dropout, fully_connected\n", "from tflearn.layers.conv import conv_2d, max_pool_2d\n", "from tflearn.layers.estimator import regression\n", "from tflearn.data_preprocessing import ImagePreprocessing\n", "from tflearn.data_augmentation import ImageAugmentation\n", "from tflearn.data_utils import shuffle\n", "import pickle"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "0556e96645d382ef79ba82ee487cf7132083f5bd", "_cell_guid": "50811239-4ffe-4a2d-b13b-0329eddabcc2", "collapsed": true}, "source": ["from keras.layers import Conv2D, MaxPooling2D, Input, Dense, Flatten, Dropout\n", "from keras.models import Model\n", "from keras.layers import concatenate"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "c5cac91b839c2fa85ac714fc5492ac457bfc95f9", "_cell_guid": "7101cbde-bb8c-49f4-b352-b48e05744e10", "collapsed": true}, "source": ["nindl=range(nind)\n", "nindb=int(ratio*nind)\n", "trY = np.asarray([stgdf[stgdf.Ind==names_file[k].replace('.aps', '')].sort_values(by=['Zone'])['Probability']\n", "                  for k in nindl[:nindb]])\n", "teY = np.asarray([stgdf[stgdf.Ind==names_file[k].replace('.aps', '')].sort_values(by=['Zone'])['Probability']\n", "                  for k in nindl[nindb:]])"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "c62239e14608888ffb184893dd9a62e871002e2e", "_cell_guid": "03ff5903-9969-4c73-89d7-00abaacd4058", "collapsed": true}, "source": ["trX = dict()\n", "tData=dict((k, data_file[k]) for k in names_file[:nindb])\n", "for i in range(16):\n", "    Xi = [value[:,:,i].transpose() for value in tData.values()]\n", "    Xi=np.asarray(Xi)/np.amax((np.abs(np.asarray(Xi))))\n", "    Xi = Xi.reshape(Xi.shape[0], 660, 512,1)\n", "    trX[i]=Xi\n", "\n", "teX = dict()\n", "tData=dict((k, data_file[k]) for k in names_file[nindb:])\n", "for i in range(16):\n", "    Xi = [value[:,:,i].transpose() for value in tData.values()]\n", "    Xi=np.asarray(Xi)/np.amax((np.abs(np.asarray(Xi))))\n", "    Xi = Xi.reshape(Xi.shape[0], 660, 512,1)\n", "    teX[i]=Xi"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "29ae0c011508d71bdec53e7f8ea73e2b5a9a0813", "_cell_guid": "cbb74323-e9ae-4750-bdba-1adb7e5992be", "collapsed": true}, "source": ["del data_file"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "483b335138b88781751146375eb5b1fe1443babe", "scrolled": true, "_cell_guid": "9fbf1b21-3b8f-4276-ad33-0bdfc5b66aa4", "collapsed": true}, "source": ["class_X = dict()\n", "for i in range(17):\n", "    Xi=dict()\n", "    Yi=trY[:,i]\n", "    zero_weight=sum(Yi)\n", "    uno_weight=Yi.shape[0]-zero_weight\n", "    zero_weight /=Yi.shape[0]\n", "    uno_weight/=Yi.shape[0]\n", "    Xi[0]=max(zero_weight/uno_weight,0.05)\n", "    Xi[1]=min(uno_weight/zero_weight,20)\n", "    class_X[int(i+1)]=Xi"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "d1825b523585cf14807420e6fc110986be937d42", "_cell_guid": "c5fbbee3-ac98-455c-b4fe-656c94c4c12f", "collapsed": true}, "source": ["class_X"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "2e11ac874febf0c977a79bf67f1aec3adbc5123a", "_cell_guid": "3c27192b-c65c-4a92-9afb-cb41606f57ff", "collapsed": true}, "source": ["class ConvNet( object ):\n", "\n", "    def parameters(self): \n", "        params_w = {'wLyr1': tf.Variable(tf.random_normal([ 5, 5, 1,  self.lyr1FilterNo_                        ])),\n", "                    'wLyr2': tf.Variable(tf.random_normal([ 3, 3,     (self.imageRot_)*self.lyr1FilterNo_ , self.lyr2FilterNo_   ])),\n", "                    'wLyr3': tf.Variable(tf.random_normal([ 3, 3,     self.lyr2FilterNo_   , self.lyr2FilterNo_   ])),\n", "                    'wFCh1':  tf.Variable(tf.random_normal([ self.LyrSize_ , self.fcHidLyrSize1_   ])),\n", "                    'wFCh2':  tf.Variable(tf.random_normal([ self.fcHidLyrSize1_           , self.fcHidLyrSize2_   ])),   \n", "                    'wFCh3':  tf.Variable(tf.random_normal([ self.fcHidLyrSize2_           , self.fcHidLyrSize3_   ])),\n", "                    'wFCh4':  tf.Variable(tf.random_normal([ self.fcHidLyrSize3_           , self.fcHidLyrSize4_   ])),\n", "                    'wFCh5':  tf.Variable(tf.random_normal([ self.fcHidLyrSize4_           , self.fcHidLyrSize5_   ])),\n", "                    'wFCh6':  tf.Variable(tf.random_normal([ self.fcHidLyrSize5_           , self.fcHidLyrSize6_   ])),\n", "                    'wOut':  tf.Variable(tf.random_normal([  self.fcHidLyrSize6_            , self.outLyrSize_     ]))}\n", "                \n", "        params_b = {'bLyr1': tf.Variable(tf.random_normal([           self.lyr1FilterNo_                        ])),\n", "                    'bLyr2': tf.Variable(tf.random_normal([           self.lyr2FilterNo_                        ])),\n", "                    'bLyr3': tf.Variable(tf.random_normal([           self.lyr2FilterNo_                        ])),\n", "                    'bFCh1':  tf.Variable(tf.random_normal([           self.fcHidLyrSize1_                        ])),\n", "                    'bFCh2':  tf.Variable(tf.random_normal([           self.fcHidLyrSize2_                        ])),\n", "                    'bFCh3':  tf.Variable(tf.random_normal([           self.fcHidLyrSize3_                        ])),\n", "                    'bFCh4':  tf.Variable(tf.random_normal([           self.fcHidLyrSize4_                        ])),\n", "                    'bFCh5':  tf.Variable(tf.random_normal([           self.fcHidLyrSize5_                        ])),\n", "                    'bFCh6':  tf.Variable(tf.random_normal([           self.fcHidLyrSize6_                        ])),\n", "                    'bOut':  tf.Variable(tf.random_normal([           self.outLyrSize_                          ]))}\n", "        return params_w,params_b\n", "\n", "    #======================================================================================================================================================================================================================    \n", "    \n", "    def scoreB(self):\n", "        \n", "        def scoreA(xi,W, b, strides=1,k=2):\n", "            x = tf.reshape(xi, shape = [-1,660,512,1])\n", "            x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='VALID')\n", "            x = tf.nn.bias_add(x, b)\n", "            x = tf.nn.relu(x)   \n", "            x= tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME')\n", "            return x\n", "        #------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------      \n", "\n", "        def conv2d(x, W, b, strides=1):\n", "            x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='VALID')\n", "            x = tf.nn.bias_add(x, b)\n", "            return x\n", "\n", "        #------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------  \n", "\n", "        def maxpool2d(x, k=2):\n", "            return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME')\n", "        #------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n", "        output1 = scoreA (self.x1_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output2 = scoreA (self.x2_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output3 = scoreA (self.x3_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output4 = scoreA (self.x4_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output5 = scoreA (self.x5_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output6 = scoreA (self.x6_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output7 = scoreA (self.x7_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output8 = scoreA (self.x8_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output9 = scoreA (self.x9_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output10 = scoreA (self.x10_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output11 = scoreA (self.x11_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output12 = scoreA (self.x12_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output13 = scoreA (self.x13_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output14 = scoreA (self.x14_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output15 = scoreA (self.x15_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        output16 = scoreA (self.x16_, self.params_w_['wLyr1'], self.params_b_['bLyr1'])\n", "        print('Output_i')\n", "        print(output1.get_shape().as_list())\n", "        concatenated=output1\n", "        for i in range(2,self.imageRot_+1):\n", "            concatenated=tf.concat([concatenated, eval('output%d'% (i))], 3)\n", "        print('Concatenated')\n", "        print(concatenated.get_shape().as_list())\n", "        \n", "        # 1)  \n", "        convLyr_12_conv = conv2d (concatenated, self.params_w_['wLyr2'], self.params_b_['bLyr2'])\n", "        convLyr_12_relu = tf.nn.relu(convLyr_12_conv)\n", "        print('Conv1')\n", "        print(convLyr_12_relu.get_shape().as_list())\n", "        convLyr_12_pool = maxpool2d(convLyr_12_relu, k=2)\n", "        print('Pool1')\n", "        print(convLyr_12_pool.get_shape().as_list())\n", "        \n", "        # 2)\n", "        convLyr_22_conv = conv2d(convLyr_12_pool, self.params_w_['wLyr3'], self.params_b_['bLyr3'])\n", "        convLyr_22_relu = tf.nn.relu(convLyr_22_conv)\n", "        print('Conv2')\n", "        print(convLyr_22_relu.get_shape().as_list())\n", "        convLyr_22_pool = maxpool2d(convLyr_22_relu, k=2)\n", "        print('Pool2')\n", "        print(convLyr_22_pool.get_shape().as_list())\n", "        \n", "        # 3)\n", "        convLyr_32_conv = conv2d(convLyr_22_pool, self.params_w_['wLyr3'], self.params_b_['bLyr3'])\n", "        convLyr_32_relu = tf.nn.relu(convLyr_32_conv)\n", "        print('Conv3')\n", "        print(convLyr_32_relu.get_shape().as_list())\n", "        convLyr_32_pool = maxpool2d(convLyr_32_relu, k=2)\n", "        print('Pool3')\n", "        print(convLyr_32_pool.get_shape().as_list())\n", "        \n", "\n", "        fcLyr_ = tf.nn.dropout(convLyr_32_pool, self.keepProb_)\n", "        print('DropOut')\n", "        print(fcLyr_.get_shape().as_list())\n", "\n", "        fcLyr_ = tf.contrib.layers.flatten(fcLyr_)\n", "        print('DropOut')\n", "        print(fcLyr_.get_shape().as_list())\n", "        self.LyrSize_ = fcLyr_.get_shape().as_list()[1]\n", "        [self.params_w_, self.params_b_] = ConvNet.parameters(self)\n", "     \n", "        # 4) Fully Connected\n", "        fcLyr_1 = tf.add(tf.matmul(fcLyr_, self.params_w_['wFCh1']), self.params_b_['bFCh1'])\n", "        fcLyr_1 = tf.nn.relu(fcLyr_1)\n", "        print('Full1')\n", "        print(fcLyr_1.get_shape().as_list())\n", "\n", "        fcLyr_2 = tf.add(tf.matmul(fcLyr_1, self.params_w_['wFCh2']), self.params_b_['bFCh2'])\n", "        fcLyr_2 = tf.nn.relu(fcLyr_2)\n", "        print('Full2')\n", "        print(fcLyr_2.get_shape().as_list())\n", "\n", "        fcLyr_3 = tf.add(tf.matmul(fcLyr_2, self.params_w_['wFCh3']), self.params_b_['bFCh3'])\n", "        fcLyr_3 = tf.nn.relu(fcLyr_3)\n", "        print('Full3')\n", "        print(fcLyr_3.get_shape().as_list())\n", "        \n", "        fcLyr_4 = tf.add(tf.matmul(fcLyr_3, self.params_w_['wFCh4']), self.params_b_['bFCh4'])\n", "        fcLyr_4 = tf.nn.relu(fcLyr_4)\n", "        print('Full4')\n", "        print(fcLyr_4.get_shape().as_list())\n", "        \n", "        fcLyr_5 = tf.add(tf.matmul(fcLyr_4, self.params_w_['wFCh5']), self.params_b_['bFCh5'])\n", "        fcLyr_5 = tf.nn.relu(fcLyr_5)\n", "        print('Full5')\n", "        print(fcLyr_5.get_shape().as_list())\n", "    \n", "        fcLyr_6 = tf.add(tf.matmul(fcLyr_5, self.params_w_['wFCh6']), self.params_b_['bFCh6'])\n", "        fcLyr_6 = tf.nn.relu(fcLyr_6)\n", "        print('Full6')\n", "        print(fcLyr_6.get_shape().as_list())\n", "\n", "        netOut = tf.add(tf.matmul(fcLyr_6, self.params_w_['wOut']), self.params_b_['bOut'])\n", "        print('Out')\n", "        print(netOut.get_shape().as_list())\n", "        return netOut\n", "        \n", "    #------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ \n", "\n", "    def costs(self):\n", "        \n", "        score_split = tf.split( axis=1, num_or_size_splits=self.outLyrSize_, value=self.score_ )\n", "        label_split = tf.split( axis=1, num_or_size_splits=self.outLyrSize_, value=self.y_  )\n", "        weight_split =tf.split( axis=0, num_or_size_splits=self.outLyrSize_, value=self.weightClass_)\n", "        \n", "        total = 0.0\n", "        for i in range (len(score_split)):\n", "            #a=weight_split[i][1]/weight_split[i][0]\n", "            #print(a)\n", "            # your class weights\n", "            #class_weights = [weight_split[i][0], weight_split[i][1]]\n", "            # deduce weights for batch samples based on their true label\n", "            #weights = tf.reduce_sum(class_weights * label_split[i], axis=1)\n", "            # compute your (unweighted) softmax cross entropy loss\n", "            unweighted_losses=tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(logits=score_split[i],\n", "                                                                    targets=label_split[i],\n", "                                                                    pos_weight=weight_split[i][0]))\n", "            #unweighted_losses = tf.losses.sparse_softmax_cross_entropy(\n", "            #    logits=score_split[i], labels=label_split[i],weights=weight_split[i][1])\n", "            # apply the weights, relying on broadcasting of the multiplication\n", "            #weighted_losses = unweighted_losses * weights\n", "            # reduce the result to get your final loss\n", "            loss = unweighted_losses#tf.reduce_mean(weighted_losses)\n", "            total += loss\n", "        return total\n", "        \n", "    #------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------   \n", "\n", "    def optimizer(self):\n", "        return tf.train.AdamOptimizer(learning_rate = self.lr_).minimize(self.cost_)\n", "    \n", "    def predicter(self):\n", "        \n", "        return [self.score_, tf.sigmoid(self.score_)]\n", "\n", "\n", "    #------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n", "\n", "    def accuracy(self): \n", "    \n", "        score_split = tf.split( axis=1, num_or_size_splits=self.outLyrSize_,value=self.score_ )\n", "        label_split = tf.split( axis=1, num_or_size_splits=self.outLyrSize_, value=self.y_     )\n", "   \n", "        correct_pred1  = tf.equal(tf.argmax(score_split[0],1), tf.argmax(label_split[0],1))  \n", "        correct_pred2  = tf.equal(tf.argmax(score_split[1],1), tf.argmax(label_split[1],1))   \n", "        correct_pred3  = tf.equal(tf.argmax(score_split[2],1), tf.argmax(label_split[2],1))  \n", "        correct_pred4  = tf.equal(tf.argmax(score_split[3],1), tf.argmax(label_split[3],1))  \n", "        correct_pred5  = tf.equal(tf.argmax(score_split[4],1), tf.argmax(label_split[4],1))  \n", "        correct_pred6  = tf.equal(tf.argmax(score_split[5],1), tf.argmax(label_split[5],1))  \n", "        correct_pred7  = tf.equal(tf.argmax(score_split[6],1), tf.argmax(label_split[6],1))  \n", "        correct_pred8  = tf.equal(tf.argmax(score_split[7],1), tf.argmax(label_split[7],1))  \n", "        correct_pred9  = tf.equal(tf.argmax(score_split[8],1), tf.argmax(label_split[8],1))  \n", "        correct_pred10  = tf.equal(tf.argmax(score_split[9],1), tf.argmax(label_split[9],1))  \n", "        correct_pred11  = tf.equal(tf.argmax(score_split[10],1), tf.argmax(label_split[10],1))  \n", "        correct_pred12  = tf.equal(tf.argmax(score_split[11],1), tf.argmax(label_split[11],1))  \n", "        correct_pred13  = tf.equal(tf.argmax(score_split[12],1), tf.argmax(label_split[12],1))  \n", "        correct_pred14  = tf.equal(tf.argmax(score_split[13],1), tf.argmax(label_split[13],1))  \n", "        correct_pred15  = tf.equal(tf.argmax(score_split[14],1), tf.argmax(label_split[14],1))  \n", "        correct_pred16  = tf.equal(tf.argmax(score_split[15],1), tf.argmax(label_split[15],1))\n", "        correct_pred17  = tf.equal(tf.argmax(score_split[16],1), tf.argmax(label_split[16],1))\n", "        \n", "        return [correct_pred1 , correct_pred2, correct_pred3, correct_pred4 ,correct_pred5, correct_pred6 , correct_pred7, correct_pred8 , correct_pred9,\n", "                correct_pred10 , correct_pred11, correct_pred12 , correct_pred13, correct_pred14 , correct_pred15, correct_pred16 , correct_pred17]\n", "        \n", "   #------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------    \n", "\n", "    def __init__(self,x,y,lr,\n", "                 lyr1FilterNo,lyr2FilterNo,\n", "                 fcHidLyrSize1,fcHidLyrSize2,fcHidLyrSize3,fcHidLyrSize4,fcHidLyrSize5,fcHidLyrSize6,\n", "                 inLyrSize,outLyrSize, keepProb, weight_class, imageRot):\n", "        \n", "        self.x1_            = x[1]\n", "        self.x2_            = x[2]\n", "        self.x3_            = x[3]\n", "        self.x4_            = x[4]\n", "        self.x5_            = x[5]\n", "        self.x6_            = x[6]\n", "        self.x7_            = x[7]\n", "        self.x8_            = x[8]\n", "        self.x9_            = x[9]\n", "        self.x10_            = x[10]\n", "        self.x11_            = x[11]\n", "        self.x12_           = x[12]\n", "        self.x13_            = x[13]\n", "        self.x14_            = x[14]\n", "        self.x15_            = x[15]\n", "        self.x16_            = x[16]\n", "\n", "        self.y_            = y\n", "        self.lr_           = lr\n", "        self.outLyrSize_   = outLyrSize\n", "        self.inLyrSize_    = inLyrSize\n", "        self.lyr1FilterNo_ = lyr1FilterNo\n", "        self.lyr2FilterNo_ = lyr2FilterNo\n", "        self.fcHidLyrSize1_ = fcHidLyrSize1\n", "        self.fcHidLyrSize2_ = fcHidLyrSize2\n", "        self.fcHidLyrSize3_ = fcHidLyrSize3\n", "        self.fcHidLyrSize4_ = fcHidLyrSize4\n", "        self.fcHidLyrSize5_ = fcHidLyrSize5\n", "        self.fcHidLyrSize6_ = fcHidLyrSize6\n", "        self.keepProb_      = keepProb\n", "        self.weightClass_   = weight_class\n", "        self.LyrSize_ =1\n", "        self.imageRot_ = imageRot\n", "\n", "        [self.params_w_, self.params_b_] = ConvNet.parameters(self) # initialization and packing the parameters\n", "        self.score_                      = ConvNet.scoreB     (self)  # Computing the score function\n", "        self.cost_                       = ConvNet.costs     (self)  # Computing the cost function\n", "        self.optimizer_                  = ConvNet.optimizer (self)  # Computing the update function\n", "        self.predicter_                  = ConvNet.predicter (self)  # Computing the update function\n", "        [self.perf_1, self.perf_2, self.perf_3, self.perf_4,self.perf_5,\n", "         self.perf_6, self.perf_7, self.perf_8,self.perf_9, self.perf_10,\n", "         self.perf_11, self.perf_12,self.perf_13, self.perf_14,self.perf_15,\n", "         self.perf_16, self.perf_17] = ConvNet.accuracy  (self)  # performance\n"], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "27af775169471e2f3f783ff5687dc1b4fc56f594", "scrolled": false, "_cell_guid": "7a9d9ef2-b36b-41c9-b10a-84d590d9efde", "collapsed": true}, "source": ["if __name__ == '__main__':\n", "    \n", "    lyr1FilterNo = 16 \n", "    lyr2FilterNo = 32 \n", "    \n", "    fcHidLyrSize1 = 15000\n", "    fcHidLyrSize2 = 7000\n", "    fcHidLyrSize3 = 2000\n", "    fcHidLyrSize4 = 600\n", "    fcHidLyrSize5 = 150\n", "    fcHidLyrSize6 = 37\n", "    inLyrSize    = 660 * 512\n", "    outLyrSize   = 17\n", "    lr           = 0.001\n", "    batch_size   = 1\n", "   \n", "    dropout      = 0.5\n", "    keepProb     = tf.placeholder(tf.float32)\n", "    x            = tf.placeholder(tf.float32)\n", "    y            = tf.placeholder(tf.float32, [None, outLyrSize])\n", "    weight_class = tf.placeholder(tf.float32)\n", "    imageRot     = 15\n", "    \n", "    ConvNet_class = ConvNet(x,y,\n", "                            lr,\n", "                            lyr1FilterNo,lyr2FilterNo,\n", "                            fcHidLyrSize1,fcHidLyrSize2,fcHidLyrSize3,fcHidLyrSize4,fcHidLyrSize5,fcHidLyrSize6,\n", "                            inLyrSize,outLyrSize,\n", "                            keepProb,\n", "                            weight_class, imageRot)\n", "    initVar = tf.global_variables_initializer()\n", "    \n", "    with tf.Session() as sess:\n", "        sess.run(initVar)  \n", "        index = 0 \n", "\n", "        for batch_i in range(4000):\n", "            trData_i=dict()\n", "            trLabel_i = []\n", "            print(index)\n", "            for i in range(16):\n", "                trXi = trX[i][index : index + batch_size]\n", "                trData_i[i]=trXi\n", "            trLabel_i.append( trY[ index : index + batch_size ] )\n", "            \n", "            index += batch_size\n", "            \n", "            if index > ( len(trX[0]) -1 ):\n", "                index = 0\n", "                \n", "\n", "            trLabel_i = np.reshape( trLabel_i, ( -1, 17 ) )\n", "            #print(trLabel_i)\n", "            sess.run( ConvNet_class.optimizer_ ,\n", "                     feed_dict = {x:[values for values in trData_i.values()],\n", "                                  y:trLabel_i,\n", "                                  keepProb:dropout,\n", "                                  weight_class : np.asarray([x for d in class_X.values() for x in d.values()])})\n", "            #print(trLabel_i)\n", "            if index== 0: \n", "            \n", "                cost_tr = sess.run(ConvNet_class.cost_,\n", "                                   feed_dict={x:[values for values in trData_i.values()],\n", "                                              y:trLabel_i,   keepProb: 1.,\n", "                                             weight_class : np.asarray([x for d in class_X.values() for x in d.values()])})\n", "                cost_te = sess.run(ConvNet_class.cost_,\n", "                                   feed_dict={x:[values for values in teX.values()],\n", "                                              y:teY,  keepProb: 1.,\n", "                                             weight_class : np.asarray([x for d in class_X.values() for x in d.values()])})\n", "                \n", "                # test accuracy\n", "                [accu1, accu2,accu3, accu4,accu5, accu6,accu7, accu8,accu9,\n", "                accu10,accu11, accu12,accu13, accu14,accu15, accu16,accu17]  = sess.run([ConvNet_class.perf_1, ConvNet_class.perf_2,\n", "                                                                                         ConvNet_class.perf_3, ConvNet_class.perf_4,\n", "                                                                                         ConvNet_class.perf_5, ConvNet_class.perf_6,\n", "                                                                                         ConvNet_class.perf_7, ConvNet_class.perf_8,\n", "                                                                                         ConvNet_class.perf_9, ConvNet_class.perf_10,\n", "                                                                                         ConvNet_class.perf_11,ConvNet_class.perf_12,\n", "                                                                                         ConvNet_class.perf_13,ConvNet_class.perf_14,\n", "                                                                                         ConvNet_class.perf_15,ConvNet_class.perf_16, \n", "                                                                                         ConvNet_class.perf_17] ,\n", "                                                                                        feed_dict={x:[values for values in teX.values()],\n", "                                                                                                   y:teY,  keepProb: 1.})\n", "                numOfposit = 0.0\n", "                for tt in range(accu1.shape[0]):\n", "                    if accu1[tt] == accu2[tt] and accu1[tt] == True:\n", "                        numOfposit += 1\n", "                test_accu = numOfposit / accu1.shape[0]\n", "\n", "                print(\"%4d, cost_tr: %4.2g , cost_te: %4.2g , testAccu: %4.2g \"% ( batch_i , cost_tr , cost_te ,test_accu ) )\n", "                if cost_tr==0:\n", "                    trData_i=dict()\n", "                    trLabel_i = []\n", "                    for i in range(16):\n", "                        trXi = trX[i][0 : 2]\n", "                        trData_i[i]=trXi\n", "                    trLabel_i.append( trY[0 : 2 ] )\n", "                    trLabel_i = np.reshape( trLabel_i, ( -1, 17 ) )\n", "                    \n", "                    [Score_te,Pred_te] = sess.run(ConvNet_class.predicter_,\n", "                       feed_dict={x:[values for values in trData_i.values()],\n", "                                  y:trLabel_i,  keepProb: 1.,\n", "                                  weight_class : np.asarray([x for d in class_X.values() for x in d.values()])})\n", "                    print(trY[:2])\n", "                    print(Pred_te[:2].astype(int))\n", "                    print(Score_te[:2])\n", "                \n", "                [Score_te,Pred_te] = sess.run(ConvNet_class.predicter_,\n", "                                              feed_dict={x:[values for values in teX.values()],\n", "                                                         y:teY,  keepProb: 1.,\n", "                                                         weight_class : np.asarray([x for d in class_X.values() for x in d.values()])})\n", "                print(teY)\n", "                print(Pred_te.astype(int))\n", "                print(Score_te)\n", "    \n", "    Pred_te = sess.run(ConvNet_class.predicter_,\n", "                       feed_dict={x:[values for values in teX.values()],\n", "                                  y:teY,  keepProb: 1.,\n", "                                  weight_class : np.asarray([x for d in class_X.values() for x in d.values()])})\n", "    print(teY)\n", "    print(Pred_te)\n", "                "], "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "012ea040595d4ed0b3756cefa5d54ac948fe4512", "_cell_guid": "208aac4e-307f-4f0f-acd3-ee4fe08b54a1", "collapsed": true}, "source": ["## "], "cell_type": "markdown"}, {"metadata": {"_uuid": "4df9b965b00b845bc97abc48e3738ec3d73c18f0", "_cell_guid": "258900ca-e28f-490b-94db-50fc63b64232", "collapsed": true}, "source": [], "outputs": [], "execution_count": null, "cell_type": "code"}]}