{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pandas as pd\nimport os\nimport tensorflow as tf\nprint(os.listdir(\"../input\"))","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true},"cell_type":"markdown","source":"# Import data"},{"metadata":{"_cell_guid":"57373136-4f82-4aca-8890-d0b0e01561d2","_uuid":"ebeab6b68afac226c09d84d50f09317785d381ff","trusted":true},"cell_type":"code","source":"for i in os.listdir('../input/'):\n    print(i)","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"d2d3ba75-8738-4e20-98df-c42ac5dd4d68","_uuid":"9b443945f6b06d9d9047444052c7f637491a78af","trusted":true},"cell_type":"code","source":"detectors_df = pd.read_csv('../input/' + 'detectors.csv')\ndetectors_df.head()","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"844b9121-71d1-4aa2-b2c6-cfdf61a96c68","_uuid":"e27d74dcdc6174f307b06369c2e0a20b3368acfe","trusted":true},"cell_type":"code","source":"def underlined(text):\n    return \"\\033[4m{}\\033[0m\".format(text)\n\nprint(underlined('Columns of detectors.csv table:'))\nfor i in list(detectors_df.columns):\n    print(i)","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"133f8e0e-f2b6-45d7-b982-efad405a10d8","_uuid":"b626dcb21d1efb58d29cfb6fd5ecab4fc31374a4","trusted":true},"cell_type":"code","source":"sample_submission_df = pd.read_csv('../input/' + 'sample_submission.csv')\nsample_submission_df.head()","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"df154985-790e-4ece-9aed-b7a8754c3198","_uuid":"8b6abf5cbd94e44104c275d4b89db2d7a160cc31","trusted":true},"cell_type":"code","source":"sample_submission_df.tail()","execution_count":6,"outputs":[]},{"metadata":{"_cell_guid":"0a2080fd-3aec-4154-ad88-d3b81b52fa19","_uuid":"a15f550497c59f0436066ca14cdf63a21fa5fec8"},"cell_type":"markdown","source":"## Train data:"},{"metadata":{"_cell_guid":"981ed0a1-5c54-47f4-a5a7-b52d18afedf9","_uuid":"03c984788a5b7b3db48448ff52b569c3f230b5f6","trusted":true},"cell_type":"code","source":"def parse_event_id(filename):\n    return int(filename[5:].split('-')[0])\n\n# Test:\nsample_filename = os.listdir('../input/train_1')[0]\nparse_event_id(sample_filename)","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"0cc13ee5-6ee7-4b8b-8c8f-7f9f6bb0db3b","scrolled":true,"_uuid":"02f64f7cea6899c724ca30d85ac89e673b71971a","trusted":true},"cell_type":"code","source":"train_filenames = os.listdir('../input/train_1')\ntrain_event_ids = np.unique(sorted([parse_event_id(i) for i in train_filenames]))\nprint('train event ids:', train_event_ids[:10],'...', train_event_ids[-10:])","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"e680a52e-e8d8-4d15-a88e-d7e0b904f087","_uuid":"2ed4b701078fc2d3319d9a1183e8547acac9f43b","trusted":true},"cell_type":"code","source":"def get_by_event_id_train(id_):\n    if id_ in train_event_ids:\n        return sorted(np.array(train_filenames)[[id_ == parse_event_id(i) for i in train_filenames]])\n    else:\n        return None\n\n# Test:\nprint(get_by_event_id_train(1000))\nprint(get_by_event_id_train(10))","execution_count":9,"outputs":[]},{"metadata":{"_cell_guid":"870ef959-c911-40c8-9a8a-46295a617b5d","_uuid":"6ea3a080237874c6059d90d68b1da263c5091c1d","collapsed":true,"trusted":true},"cell_type":"code","source":"cells_df = pd.read_csv('../input/train_1/'+get_by_event_id_train(1000)[0]) \nhits_df = pd.read_csv('../input/train_1/'+get_by_event_id_train(1000)[1])\nparticles_df = pd.read_csv('../input/train_1/'+get_by_event_id_train(1000)[2])\ntruth_df = pd.read_csv('../input/train_1/'+get_by_event_id_train(1000)[3])","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"4084e6cc-6d59-4422-b763-11ea9f7938f6","_uuid":"15c91b1e82166c6eab352c98509fa098b1c84519","trusted":true},"cell_type":"code","source":"cells_df.head()","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"9a61dc34-8550-465f-a85c-629f175ae5de","_uuid":"c6c241bdc5df078c731b36fab518454810ec1c75","trusted":true},"cell_type":"code","source":"hits_df.head()","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"1b23cc20-9a7d-4cfc-8cf3-d1b3f3dd5bdb","_uuid":"3817239c0651484872e405f0e6dcb1cfdd7655e6","trusted":true},"cell_type":"code","source":"# Particles origin\nparticles_df.head()","execution_count":13,"outputs":[]},{"metadata":{"_cell_guid":"7b54914f-4309-489f-9555-00e97e7ac980","_uuid":"68a921958e102ae225f5a959edc0510442b81c89","trusted":true},"cell_type":"code","source":"# Link hit_id / particle_id\ntruth_df.head()","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"320de403-8d85-43ff-95c8-17684d1df9d1","_uuid":"e9912874bbc6a8bb069b812c162cd32c25f980cc"},"cell_type":"markdown","source":"## Test data"},{"metadata":{"_cell_guid":"75c266f1-2cb7-4eb4-85c1-1903aabf3430","scrolled":true,"_uuid":"9763c0aa29a2200c49682a872d900acc9a95561d","trusted":true},"cell_type":"code","source":"test_filenames = os.listdir('../input/test/')\ntest_event_ids = np.unique(sorted([parse_event_id(i) for i in test_filenames]))\nprint('test event ids:', test_event_ids[:10],'...', test_event_ids[-10:])","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"1b15e908-c141-4153-8f14-404570c1a9e5","_uuid":"66170cbdd451624951b9266e2c34293d60a389fc","trusted":true},"cell_type":"code","source":"def get_by_event_id_test(id_):\n    if id_ in test_event_ids:\n        return sorted(np.array(test_filenames)[[id_ == parse_event_id(i) for i in test_filenames]])\n    else:\n        return None\n\n# Test:\nprint(get_by_event_id_test(1000))\nprint(get_by_event_id_test(10))","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"1bf2279c-e692-416a-bb6b-e807807efbde","_uuid":"c33fcc23dbdb314c4c2e0968a236d102b0e0cb3d","collapsed":true,"trusted":true},"cell_type":"code","source":"test_cells_df = pd.read_csv('../input/test/' + get_by_event_id_test(10)[0])\ntest_hits_df = pd.read_csv('../input/test/' + get_by_event_id_test(10)[1])","execution_count":17,"outputs":[]},{"metadata":{"_cell_guid":"26734f95-3cfa-4612-a9de-9c3acef22ea9","_uuid":"d4601a4668b6b8b1fcbd434e5287344f6e11f978","trusted":true},"cell_type":"code","source":"test_cells_df.head()","execution_count":18,"outputs":[]},{"metadata":{"_cell_guid":"34c6cbb4-3d52-43a8-b87a-820e7f2e54c7","scrolled":true,"_uuid":"234398765c1f997809d7c5c71c3f641ac896e88b","trusted":true},"cell_type":"code","source":"test_hits_df.head()","execution_count":19,"outputs":[]},{"metadata":{"_cell_guid":"be255932-458f-4ca1-b637-41966e24395d","_uuid":"e52700e1b609b1353bef2648400c42ce5a279f5f"},"cell_type":"markdown","source":"## Processing"},{"metadata":{"_cell_guid":"c45c7dd7-b1e8-475c-97e5-f574b2188975","_uuid":"9f90835820522009022280adc41019035afda213","collapsed":true,"trusted":true},"cell_type":"code","source":"# Reindex hits_df\nhits_df.index = hits_df.hit_id\nhits_df = hits_df.drop('hit_id', axis=1)","execution_count":20,"outputs":[]},{"metadata":{"_cell_guid":"51fe2298-6eba-4cf9-bd76-a29d5672d243","_uuid":"dc2649aef3af9848b076ccc8bff99ccd52eb7dde","trusted":true},"cell_type":"code","source":"print('Len:', len(cells_df))\ncells_df.head()","execution_count":21,"outputs":[]},{"metadata":{"_cell_guid":"4e6d52a1-cab6-4b2c-a9a9-14fdc9cc2111","_uuid":"df09ccf6b45bccd14eba42ba6b6fd5fa7c25ef87","trusted":true},"cell_type":"code","source":"print('Len:', len(hits_df))\nhits_df.head()","execution_count":22,"outputs":[]},{"metadata":{"_cell_guid":"23973186-2d03-4707-8dd0-2549f9f7a791","_uuid":"7e367a7ae687d6d4670742409a4344a4af38de78","trusted":true},"cell_type":"code","source":"truth_df.head()","execution_count":23,"outputs":[]},{"metadata":{"_cell_guid":"21be8016-c3cc-4271-aa54-d251e8f01e3f","_uuid":"88a874107670d2242f2cf4a74070a941a257b370","trusted":true},"cell_type":"code","source":"sample_particle_id = 0\nwhile sample_particle_id == 0:\n    sample_particle_id = int(truth_df.sample()['particle_id'])\nprint(sample_particle_id)","execution_count":24,"outputs":[]},{"metadata":{"_cell_guid":"3fbcae74-a03f-4ccd-a95c-69034a8aa2ea","_uuid":"187a524e965db2c98af29eb03d3b38e7c3617c4a","collapsed":true,"trusted":true},"cell_type":"code","source":"hits_flow = np.array(truth_df[truth_df['particle_id']==sample_particle_id]['hit_id'])","execution_count":25,"outputs":[]},{"metadata":{"_cell_guid":"82657e66-4699-4c47-bfa4-8bc03d77cf1e","_uuid":"a05449e97eb2cc5bbbe57b0f57bc0e1e6161d441","trusted":true},"cell_type":"code","source":"data = np.array(truth_df[truth_df['particle_id']==sample_particle_id][['tx', 'ty', 'tz']])\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\nfig = plt.figure()\nax = fig.add_subplot(111, projection='3d')\nax.scatter(data[:,0], data[:,1], data[:,2] , c='red', marker='o', edgecolor='k')\nax.plot(data[:,0], data[:,1], data[:,2] , '-', lw=3, alpha=0.4)\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nfig.tight_layout()\nplt.show()","execution_count":26,"outputs":[]},{"metadata":{"_cell_guid":"bed77da4-d39d-4673-a229-16b1c235797e","_uuid":"facaa393fe33bc48f9e629189a7cf9fd30919eb3","trusted":true},"cell_type":"code","source":"hits_df.loc[hits_flow]","execution_count":27,"outputs":[]},{"metadata":{"_cell_guid":"c448d1c3-5cf2-4d0d-8868-b21c2e7eb00e","_uuid":"af05f48c94ab1f6fcfba0b871548b78495eefbeb","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\nfig = plt.figure(figsize=(10, 10))\n\nax = fig.add_subplot(111, projection='3d')\n\ndata = np.array(hits_df.loc[hits_flow][['x', 'y', 'z']])\nax.scatter(data[:,0], data[:,1], data[:,2] , c='red', marker='o', edgecolor='k', s=np.ones(len(data))*50)\nax.plot(data[:,0], data[:,1], data[:,2] , 'k-', lw=3, alpha=0.4)\n\ndata = np.array(hits_df[['x', 'y', 'z']].sample(100))\nax.scatter(data[:,0], data[:,1], data[:,2] , c='blue', marker='o', edgecolor='k')\n\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nax.view_init(elev=20)\nfig.tight_layout()\nplt.show()","execution_count":28,"outputs":[]},{"metadata":{"_cell_guid":"49978b79-6554-4f86-9237-1f7e2083e8ee","_uuid":"712533f9b03d2c99f699b9d0b0afea5e1a01e473","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\nfig = plt.figure(figsize=(15, 10))\n\nelev_range = [int(np.random.random()*90) for i in range(0, 3)]\nazim_range = [int(np.random.random()*180) for i in range(0, 3)]\n\nfor i, elev_angle in enumerate(elev_range):\n    for j, azim_angle in enumerate(azim_range):\n    \n        ax = fig.add_subplot(331 + i*3 + j, projection='3d')\n\n        data = np.array(hits_df.loc[hits_flow][['x', 'y', 'z']])\n        ax.scatter(data[:,0], data[:,1], data[:,2] , c='red', marker='o', edgecolor='k', s=np.ones(len(data))*50)\n        ax.plot(data[:,0], data[:,1], data[:,2] , 'k-', lw=3, alpha=0.4)\n\n        data = np.array(hits_df[['x', 'y', 'z']].sample(200))\n        ax.scatter(data[:,0], data[:,1], data[:,2] , c='blue', marker='o', edgecolor='k', alpha=0.3)\n\n        ax.set_xlabel('X Label')\n        ax.set_ylabel('Y Label')\n        ax.set_zlabel('Z Label')\n        ax.view_init(elev=elev_angle, azim=azim_angle)\n    \nfig.tight_layout()\nplt.show()","execution_count":29,"outputs":[]},{"metadata":{"_cell_guid":"788680c6-86e5-44db-8caf-9d479d714417","_uuid":"f5549f6ef700b4994ef0d4ef02280f5116462b14","trusted":true},"cell_type":"code","source":"hits_df.head()","execution_count":30,"outputs":[]},{"metadata":{"_cell_guid":"80bc66e1-4b0a-4b4b-9f98-9fcdd970fefd","_uuid":"f4cb1f25a88c9e61d7ca3a99cf56b9cbfd664d3b","collapsed":true,"trusted":true},"cell_type":"code","source":"cells_df.index = cells_df.hit_id\ncells_df = cells_df.drop('hit_id', axis=1)","execution_count":31,"outputs":[]},{"metadata":{"_cell_guid":"3bde3249-8fdc-4bf3-8565-a0a936a05414","scrolled":true,"_uuid":"d7da4ef1ee9bfaba550a841ec91e2b23bf22fd9f","trusted":true},"cell_type":"code","source":"cells_df.join(hits_df).head()","execution_count":32,"outputs":[]},{"metadata":{"_cell_guid":"0a8d6ceb-10ca-4b2f-be42-cdb973f01b2f","_uuid":"409eeb256f6c2504c13ab57bfc21b59135309660","collapsed":true,"trusted":true},"cell_type":"code","source":"truth_df.index = truth_df.hit_id\ntruth_df = truth_df.drop('hit_id', axis=1)","execution_count":33,"outputs":[]},{"metadata":{"_cell_guid":"fad4638e-0b04-49f9-8a7f-3c287968227b","scrolled":true,"_uuid":"a00cf03f646412055326f282749fc6c4af272bad","collapsed":true,"trusted":true},"cell_type":"code","source":"global_df = cells_df.join(hits_df).join(truth_df)","execution_count":34,"outputs":[]},{"metadata":{"_cell_guid":"a1f3c4fc-5ec0-4da1-bc4d-74c6fed2e3df","_uuid":"4747f974d6803fa489a6bce502c63acf12a47868","trusted":true},"cell_type":"code","source":"global_df.sort_values('particle_id').head(10)","execution_count":35,"outputs":[]},{"metadata":{"_cell_guid":"3b1ef68d-664a-4402-a84e-966435ef1b37","_uuid":"d6f25aef40681a85b3c3bd029a3fe1737b8850a5","trusted":true},"cell_type":"code","source":"global_df[['value', 'weight']].corr()","execution_count":36,"outputs":[]},{"metadata":{"_cell_guid":"936384bb-1c45-48cb-86d1-836b436f494b","_uuid":"7f6fc3512347dad81f40146767b93f97688f4601","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\nfig = plt.figure(figsize=(15, 10))\n\nelev_range = [int(np.random.random()*90) for i in range(0, 3)]\nazim_range = [int(np.random.random()*180) for i in range(0, 3)]\n\nfor i, elev_angle in enumerate(elev_range):\n    for j, azim_angle in enumerate(azim_range):\n    \n        ax = fig.add_subplot(331 + i*3 + j, projection='3d')\n\n        data = np.array(global_df[global_df['weight']!=0][['x', 'y', 'z']])[:200]\n        ax.scatter(data[:,0], data[:,1], data[:,2] , c='red', marker='o', edgecolor='k', s=np.ones(len(data))*50)\n        \n        ax.set_xlabel('X Label')\n        ax.set_ylabel('Y Label')\n        ax.set_zlabel('Z Label')\n        ax.view_init(elev=elev_angle, azim=azim_angle)\n    \nfig.tight_layout()\nplt.show()","execution_count":37,"outputs":[]},{"metadata":{"_cell_guid":"7204e1c2-ed6d-48e7-b1a5-9480fe720b2c","_uuid":"a84701f9af0b2ec59e7b99e3c17553a7fc6053d9","collapsed":true,"trusted":true},"cell_type":"code","source":"sample_particle = int(global_df[global_df['weight']!=0].sample()['particle_id'])","execution_count":38,"outputs":[]},{"metadata":{"_cell_guid":"359e68fa-1a00-4a6b-b48b-244ba025e460","scrolled":true,"_uuid":"12a96b43402e23ad12e4047927b49f8e412dc852","trusted":true},"cell_type":"code","source":"global_df[global_df['particle_id'] == sample_particle].head()","execution_count":39,"outputs":[]},{"metadata":{"_cell_guid":"03aa9ee5-2538-4b36-be9a-efe0858c6a29","_uuid":"8f43d0208ef437c1e247978143e50e7f6676fbdf"},"cell_type":"markdown","source":"### Prediction pipeline :"},{"metadata":{"_cell_guid":"05c66237-004f-4647-beb2-c11ff02ef51b","_uuid":"056647fa66f74d544ff225280e25f2c76c0368c8","collapsed":true,"trusted":true},"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn import preprocessing, cluster","execution_count":40,"outputs":[]},{"metadata":{"_cell_guid":"d06eb784-5352-46d9-b897-df228799377b","_uuid":"6dd449dca8076c2f5dbd193008685e8242a93523","collapsed":true,"trusted":true},"cell_type":"code","source":"scl = preprocessing.StandardScaler()\ndbscan = cluster.DBSCAN(eps=0.0076, min_samples=1, algorithm='kd_tree', n_jobs=-1)","execution_count":41,"outputs":[]},{"metadata":{"_cell_guid":"ab7a7707-deec-4960-ab53-13f665fd933e","_uuid":"b643b8fe6d2370e10acab9978e1226a12bd6e1d1","collapsed":true,"trusted":true},"cell_type":"code","source":"# Normalisation des points\nx = hits_df.x.values\ny = hits_df.y.values\nz = hits_df.z.values\n\nr = np.sqrt(x**2+y**2+z**2)\n\nx2 = x/r\ny2 = y/r\n\nr2 = np.sqrt(x**2+y**2)\n\nz2 = z/r2","execution_count":42,"outputs":[]},{"metadata":{"_cell_guid":"98bd52f9-eead-4276-a010-d8da45d0c922","_uuid":"0cfd845fd60ac69870573e677225e2d8d2299346","collapsed":true,"trusted":true},"cell_type":"code","source":"def get_features(dataframe, theta=0):\n    \n    x = dataframe.x.values\n    y = dataframe.y.values\n    z = dataframe.z.values\n\n    r = np.sqrt(x**2+y**2+z**2)\n    x2 = x/r\n    y2 = y/r\n    r2 = np.sqrt(x**2+y**2)\n    z2 = z/r2\n    \n    dataframe['x2'] = x2\n    dataframe['y2'] = y2\n    dataframe['z2'] = z2\n    dataframe['r'] = r2\n    \n    return dataframe","execution_count":43,"outputs":[]},{"metadata":{"_cell_guid":"3c3db33c-fe29-40f1-9a7d-f496785bbba9","_uuid":"9e63409de31f458bff1f73f374d600245890e13e","collapsed":true,"trusted":true},"cell_type":"code","source":"transformed_hits_df = get_features(hits_df)","execution_count":44,"outputs":[]},{"metadata":{"_cell_guid":"9ac5d4d2-715d-4d97-ae98-757fe751335b","_uuid":"1152b87ebc967d546173e294e7a0731cd777338e","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\n\nfig = plt.figure(figsize=(15, 10))\n\nsub_sample = transformed_hits_df.sample(4000)\n\nax = fig.add_subplot(121, projection='3d')\ndata = np.array(sub_sample[['x', 'y', 'z']])\nax.scatter(data[:,0], data[:,1], data[:,2] , c='red', marker='o', edgecolor='k', s=np.ones(len(data))*50)\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nax.view_init(elev=20, azim=0)\n\nax = fig.add_subplot(122, projection='3d')\ndata = np.array(sub_sample[['x2', 'y2', 'z2']])\nax.scatter(data[:,0], data[:,1], data[:,2] , c='red', marker='o', edgecolor='k', s=np.ones(len(data))*50)\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nax.view_init(elev=20, azim=0)\n    \nfig.tight_layout()\nplt.show()","execution_count":45,"outputs":[]},{"metadata":{"_cell_guid":"d7a02abf-e5b3-4999-afaf-e08298ca4334","scrolled":false,"_uuid":"d8a7543c80456ca3d157887addb6fb4ae9485738","trusted":true},"cell_type":"code","source":"sample_particle_id = 0\nwhile sample_particle_id == 0:\n    sample_particle_id = int(truth_df.sample()['particle_id'])\nprint('Choice of a random particle: {}'.format(sample_particle_id))\n\n# Retreive all hit from sample_particle_id\nprint('Transformed 3D DF:')\ndisplay(transformed_hits_df.loc[list(truth_df[truth_df['particle_id']==sample_particle_id].index)].head())\n\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\n\nfig = plt.figure(figsize=(15, 10))\n\n# Before transformation\nax = fig.add_subplot(121, projection='3d')\n\nsub_sample = transformed_hits_df.loc[list(truth_df[truth_df['particle_id']==sample_particle_id].index)]\ndata = np.array(sub_sample[['x', 'y', 'z']])\nax.scatter(data[:,0], data[:,1], data[:,2] , c='red', marker='o', edgecolor='k', s=np.ones(len(data))*50, label='Same track particles')\n\nsub_sample = transformed_hits_df.sample(100)\ndata = np.array(sub_sample[['x', 'y', 'z']])\nax.scatter(data[:,0], data[:,1], data[:,2] , c='yellow', marker='o', edgecolor='k', s=np.ones(len(data))*30, label='Sample particles')\n\nax.set_title('3D Plot Before Transformation')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nax.view_init(elev=45, azim=0)\nax.legend()\n\n\n# After transformation\nax = fig.add_subplot(122, projection='3d')\n\nsub_sample = transformed_hits_df.loc[list(truth_df[truth_df['particle_id']==sample_particle_id].index)]\ndata = np.array(sub_sample[['x2', 'y2', 'z2']])\nax.scatter(data[:,0], data[:,1], data[:,2] , c='red', marker='o', edgecolor='k', s=np.ones(len(data))*50, label='Same track particles')\n\nsub_sample = transformed_hits_df.sample(100)\ndata = np.array(sub_sample[['x2', 'y2', 'z2']])\nax.scatter(data[:,0], data[:,1], data[:,2] , c='yellow', marker='o', edgecolor='k', s=np.ones(len(data))*30, label='Sample particles')\n\nax.set_title('3D Plot After Transformation')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nax.view_init(elev=45, azim=0)\nax.legend()\n    \nfig.tight_layout()\nplt.show()","execution_count":46,"outputs":[]},{"metadata":{"_cell_guid":"70494e07-44c2-48ca-9169-04e848fedfbc","_uuid":"e9bb60b911cd13d02c75df55306b176a0eb876b0","trusted":true},"cell_type":"code","source":"transformed_hits_df.head()","execution_count":47,"outputs":[]},{"metadata":{"_cell_guid":"6f7b0a92-bdb7-4eb3-a252-de2d531ce0f9","_uuid":"7d35405e911f104001604bcdc150cc4e143ca020","trusted":true,"collapsed":true},"cell_type":"code","source":"link_table = transformed_hits_df.join(truth_df[['particle_id']])[['x2', 'y2', 'z2', 'particle_id']]","execution_count":48,"outputs":[]},{"metadata":{"_cell_guid":"db3c19b0-59e1-4f19-9bc1-565b2818f809","_uuid":"212059c7e946f1296c69a9239434614a03545b62","trusted":true},"cell_type":"code","source":"link_table.head()","execution_count":49,"outputs":[]},{"metadata":{"_cell_guid":"72217543-7f4e-4054-97e5-5d98eb876929","scrolled":true,"_uuid":"1c23d19b983ab2277d8f44b6f4efc20d70c28d4a","trusted":true},"cell_type":"code","source":"print(len(link_table[link_table['particle_id']==0]))\nprint(len(link_table[link_table['particle_id']!=0]))\n","execution_count":50,"outputs":[]},{"metadata":{"_cell_guid":"1fd041b9-f3f8-4402-a110-0ea8bf1f77c9","_uuid":"1b8495d72604702eac6e6faa6f6f307928c85639","trusted":true},"cell_type":"code","source":"sub_sample = link_table[link_table['particle_id']!=0]\n\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\n\nfig = plt.figure(figsize=(15, 15))\nelev_range = [0, 10, 90]\nazim_range = [0, 10, 90]\n\nfor i, elev_angle in enumerate(elev_range):\n    for j, azim_angle in enumerate(azim_range):\n\n        ax = fig.add_subplot(331 + i*3 + j, projection='3d')\n        for sample_particle_id in np.unique(sub_sample['particle_id'].values)[:100]:\n            data = np.array(sub_sample[sub_sample['particle_id']==sample_particle_id][['x2', 'y2', 'z2']])\n            ax.scatter(data[:,0], data[:,1], data[:,2], marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)\n        ax.set_title('3D Plot Before Transformation')\n        ax.set_xlabel('X Label')\n        ax.set_ylabel('Y Label')\n        ax.set_zlabel('Z Label')\n        ax.view_init(elev=elev_angle, azim=azim_angle)\n        ax.legend()\n\nfig.tight_layout()\nplt.show()\n","execution_count":51,"outputs":[]},{"metadata":{"_cell_guid":"4e50997b-11a5-4f97-a03c-ad98014a2fa0","_uuid":"84ee04d0f3b899ddffea260afef1bb2c148fb30a","trusted":true},"cell_type":"code","source":"sub_sample = link_table[link_table['particle_id']!=0]\n\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\n\nfig = plt.figure(figsize=(15, 15))\nax = fig.add_subplot(111, projection='3d')\nfor sample_particle_id in np.unique(sub_sample['particle_id'].values)[:100]:\n    data = np.array(sub_sample[sub_sample['particle_id']==sample_particle_id][['x2', 'y2', 'z2']])\n    ax.plot(data[:,0], data[:,1], data[:,2], '-', alpha=0.5, lw=4)\n    ax.scatter(data[:,0], data[:,1], data[:,2], marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)\n    \n    \nax.set_title('3D Plot Before Transformation')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nax.view_init(elev=90, azim=0)\nax.legend()\n\nfig.tight_layout()\nplt.show()","execution_count":52,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"15f2a57f570215934a5400a260ed616ae7c62c69","_cell_guid":"c95ef148-edaf-45cc-97c1-bbc51327815d","trusted":true},"cell_type":"code","source":"sub_sample = link_table[link_table['particle_id']!=0]\nfor sample_particle_id in np.unique(sub_sample['particle_id'].values)[:100]:\n    data = np.array(sub_sample[sub_sample['particle_id']==sample_particle_id][['x2', 'y2', 'z2']])","execution_count":53,"outputs":[]},{"metadata":{"_cell_guid":"2000306b-8dd4-4b0e-857c-781932bf1f85","_uuid":"e939e2003bb9679833357eddf7d9ea0d0d9ba449","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.title('Histograms of number of particles')\nplt.hist(sub_sample.groupby(['particle_id'])['x2'].count(), edgecolor='k', bins=20)\nplt.xlabel('Number of connected components')\nplt.grid(axis='y')\nplt.show()","execution_count":54,"outputs":[]},{"metadata":{"_cell_guid":"d4a72c2b-81d4-4f25-899e-2e9b48774694","_uuid":"fb82518fc2c747b01e0725c34d5c15a82affaf83","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.title('Histograms of z amplitude')\nplt.hist((sub_sample.groupby(['particle_id']).max() - sub_sample.groupby(['particle_id']).min())['z2'].values, edgecolor='k', bins=20, log=True)\nplt.xlabel('z axis amplitude of connected components')\nplt.show()","execution_count":55,"outputs":[]},{"metadata":{"_cell_guid":"5cf9348e-bc99-41b3-bb50-fd43a0c24fe7","_uuid":"5074ace1c3e67545cedf892f950fcace248e29ec","trusted":true},"cell_type":"code","source":"# Radius of projected coordinates on (O, x, y)\nsub_sample['r2'] = np.sqrt(sub_sample['x2']**2+sub_sample['y2']**2)","execution_count":56,"outputs":[]},{"metadata":{"_cell_guid":"28fabd2b-461c-43eb-a90d-35e12cae0212","_uuid":"3949083333d57ff6a08cbd7f7f3311b790421b6d","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.title('Radius (Projected on (O, x, y)) amplitude for connected components')\nplt.hist(sub_sample.groupby(['particle_id'])['r2'].max() - sub_sample.groupby(['particle_id'])['r2'].min(), bins=20, edgecolor='k', log=True)\nplt.xlabel('Radius')\nplt.show()","execution_count":57,"outputs":[]},{"metadata":{"_cell_guid":"0a8c6b0d-c6df-4c79-bf03-332d90f7791e","_uuid":"a04bf63e9c897bd37eaafd38e1cedf9fa67d1ad6","trusted":true},"cell_type":"code","source":"def loss(x, lambd=0):\n    if x > 0:\n        return max(0, x-lambd)\n    else:\n        return max(0, -(lambd+x))\n\nx = np.linspace(-10, 10, 100)\ny = [loss(x_, 2) for x_ in x]\n\nplt.title('Loss function')\nplt.axhline(0, color='black', alpha=0.5)\nplt.axvline(0, color='black', alpha=0.5)\nplt.xlabel('x')\nplt.ylabel('loss')\nplt.grid()\nplt.plot(x, y, label='loss')\nplt.legend()\nplt.show()","execution_count":58,"outputs":[]},{"metadata":{"_cell_guid":"fca65961-af12-4db7-b1d7-1ce741d1db18","_uuid":"6dbb2e34694d48dbe78118f4bf29fb9961b0367b","trusted":true,"collapsed":true},"cell_type":"code","source":"from math import atan, pi","execution_count":59,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"f335641728bb31d30d537b9b642c28106b7158ce","_cell_guid":"cc60ed91-c67f-4942-b330-292764516703","trusted":true},"cell_type":"code","source":"def get_angle(x,y):\n    \"\"\"\n    Return angle in degrees from cartesian coordinates.\n    \"\"\"\n    if x > 0:\n        return atan(y/x)\n    else:\n        return pi - atan(y/x)","execution_count":60,"outputs":[]},{"metadata":{"_cell_guid":"e9f62081-7251-4732-b4fe-ccb94560de57","_uuid":"80ff6e7b54a36cb9c6064cf75ac3f0e7ec860078","trusted":true},"cell_type":"code","source":"sub_sample['r'] = np.sqrt(sub_sample['x2']**2+sub_sample['y2']**2)\nsub_sample['theta'] = sub_sample.apply(lambda row: get_angle(row['x2'], row['y2']) - pi/2, axis=1)  # ?","execution_count":61,"outputs":[]},{"metadata":{"_cell_guid":"01ddfae6-7ba3-4272-8563-dfbe926da685","_uuid":"5e70eba16242f4e3ac45a6a3b143561350ceeb64","trusted":true,"collapsed":true},"cell_type":"code","source":"# sub_sample values are bounded between 180° and -180°","execution_count":62,"outputs":[]},{"metadata":{"_cell_guid":"99991611-038d-4978-bddc-070be4091d80","_uuid":"c9eece18f2a88f49cb80e553d865ed639fdc764f","trusted":true,"collapsed":true},"cell_type":"code","source":"def dist(point1, point2):\n    return  loss(abs(float(point1['theta'])-float(point2['theta']))%360,50)+\\\n            loss(abs(float(point1['r'])-float(point2['r'])),10)+\\\n            loss(abs(float(point1['z2'])-float(point2['z2'])),10)","execution_count":63,"outputs":[]},{"metadata":{"_cell_guid":"b89ce774-39e0-461d-bf8c-02f53ab376e8","_uuid":"d0670849c6cc22a86fb8619960d27a3fb9e30324","trusted":true,"collapsed":true},"cell_type":"code","source":"from math import sin","execution_count":64,"outputs":[]},{"metadata":{"_cell_guid":"d222784c-9bb2-41fc-8503-10ccc556cdb1","_uuid":"561dd0a83957146c13265321d47ff49eea04d08a","trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\nfor sample_particle_id in np.unique(sub_sample['particle_id'].values)[:100]:\n    data = np.array(sub_sample[sub_sample['particle_id']==sample_particle_id][['r', 'theta']])\n    #ax.plot(data[:,0]*np.cos(data[:,1]), data[:,0]*np.sin(data[:,1]), '-', alpha=0.5, lw=4)\n    ax.scatter(data[:,0]*np.cos(data[:,1]), data[:,0]*np.sin(data[:,1]), marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)\n    \nax.set_title('3D Plot Before Transformation')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.legend()\n\nfig.tight_layout()\nplt.show()","execution_count":65,"outputs":[]},{"metadata":{"_cell_guid":"d825ce94-d8cf-4a22-920a-b7604718c00f","scrolled":true,"_uuid":"d79629fa91220c8e994754f4487b1f0efdda3e84","trusted":true},"cell_type":"code","source":"# Apply rotation/compression to a segment of the pie chart\ntable_ = sub_sample[sub_sample['theta'] < pi/2][sub_sample['theta'] > 0] # Centered on pi/4","execution_count":87,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0aa09252191d8f3aff6457a6f3d8bd9dc215370a"},"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\nfor sample_particle_id in np.unique(table_['particle_id'].values)[:100]:\n    data = np.array(table_[table_['particle_id']==sample_particle_id][['r', 'theta']])\n    ax.plot(data[:,0]**2*np.cos(data[:,1]), data[:,0]**2*np.sin(data[:,1]), '-', alpha=0.5, lw=4)\n    ax.scatter(data[:,0]**2*np.cos(data[:,1]), data[:,0]**2*np.sin(data[:,1]), marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)\n    \nax.set_title('3D Plot Before Transformation')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.legend()\n\nfig.tight_layout()\nplt.show()","execution_count":75,"outputs":[]},{"metadata":{"_cell_guid":"6b3e6c83-c2f7-4dee-b29a-1fbe6c76bba7","_uuid":"3379866013abd8f5dabdfc27116c39b9c95813ef","trusted":true},"cell_type":"code","source":"center = pi/4\nprint(center)\namplitude = pi/4\nx = np.linspace(center - amplitude, center + amplitude, 100)\ny = x-np.tanh((x-center)*2)*center\nplt.plot(x, y)\nplt.axhline(0, color='black')\nplt.axvline(center - amplitude, color='black')\nplt.axvline(center + amplitude, color='black')\nplt.axhline(center, color='black')\nplt.grid()","execution_count":67,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"87840e2f59680ba68fb65c4fa3fb424af207c430","_cell_guid":"68413864-7f51-44ee-8d71-b5c92747238e","trusted":true},"cell_type":"code","source":"from math import tanh","execution_count":68,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba7ddbab4bb26a70ba764eeaf111bc1b5c10ffab"},"cell_type":"code","source":"# Apply rotation/compression to a segment of the pie chart\ntable_ = sub_sample[sub_sample['theta'] < pi/2][sub_sample['theta'] > 0] # Centered on pi/4","execution_count":116,"outputs":[]},{"metadata":{"_cell_guid":"97601930-8dde-428f-ba5d-ac1b43904e35","_uuid":"ee92afe2d6fe0f1289254af71acf4c6431d02789","trusted":true,"scrolled":false},"cell_type":"code","source":"fig = plt.figure(figsize=(15, 10))\n\n\nax = fig.add_subplot(131)\nfor sample_particle_id in np.unique(table_['particle_id'].values)[:100]:\n    data = np.array(table_[table_['particle_id']==sample_particle_id][['r', 'theta']])\n    ax.plot(data[:,0]**2*np.cos(data[:,1]), data[:,0]**2*np.sin(data[:,1]), '-', alpha=0.5, lw=2)\n    ax.scatter(data[:,0]**2*np.cos(data[:,1]), data[:,0]**2*np.sin(data[:,1]), marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)\n    \nax.set_title('3D Plot Before Transformation')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\n\ntable_['theta'] = table_['theta']*(1-table_['r']/table_['r'].max()) + (table_['theta'] - (np.tanh((table_['theta']-center)*2)*center).values)*(table_['r']/table_['r'].max())\n\nax = fig.add_subplot(132)\nfor sample_particle_id in np.unique(table_['particle_id'].values)[:100]:\n    data = np.array(table_[table_['particle_id']==sample_particle_id][['r', 'theta']])\n    x = data[:,0]**2*np.cos(data[:,1])\n    y = data[:,0]**2*np.sin(data[:,1])\n    ax.plot(x, y, '-', alpha=0.5, lw=2)\n    ax.scatter(x, y, marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)\n\nax = fig.add_subplot(133)\nfor sample_particle_id in np.unique(table_['particle_id'].values)[:100]:\n    data = np.array(table_[table_['particle_id']==sample_particle_id][['r', 'theta']])\n    x = data[:,0]**2*np.cos(data[:,1])\n    y = data[:,0]**2*np.sin(data[:,1])\n    fish_eye = [0, 0]\n    ax.plot(np.log(abs(x-fish_eye[0])), np.log(abs(y-fish_eye[1])), '-', alpha=0.5, lw=2)\n    ax.scatter(np.log(abs(x-fish_eye[0])), np.log(abs(y-fish_eye[1])), marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)    \n    \nax.set_title('3D Plot Before Transformation')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.legend()\n\nfig.tight_layout()\nplt.show()","execution_count":109,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7904560fc834eba7f1e9e81b36d2920769d22ae1"},"cell_type":"code","source":"from sklearn import metrics\nfrom sklearn.cluster import DBSCAN\n\neps_range = np.linspace(0.01, 0.1, 5)\nmin_samples_range = np.arange(1, 5)\n\nfor eps_ in eps_range:\n    for min_samples_ in min_samples_range:\n\n        dbs = DBSCAN(eps=eps_, min_samples=min_samples_)\n\n        X = table_.drop('particle_id', axis=1)\n        y = table_['particle_id']\n        y_preds = dbs.fit_predict(X)\n    \n        print('eps:{}, min_samples:{}, score:{}'.format(eps_, min_samples_, metrics.adjusted_mutual_info_score(y, y_preds)))","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}