{"cells":[{"metadata":{"_uuid":"d620b9fa8b8cb0ad43c8b5919d5ad7023068ba78"},"cell_type":"markdown","source":"**# original author: Heng CherKeng, code derived from https://www.kaggle.com/danieleewww/heng-ck-s-python-for-grzegorz-s-s-r/comments\n\n\n**# Forked from https://www.kaggle.com/sionek/mod-dbscan-0-3472/comments#332843\n**# This Python 3 environment comes with many helpful analytics libraries installed\n**# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n**# For example, here's several helpful packages to load in\n********"},{"metadata":{"_cell_guid":"e081740e-8169-4481-b1df-f5dd5488314f","_uuid":"0bee86255243664f24e4bcf48af2228a3100a8b7","trusted":true,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport os\n\nfrom trackml.dataset import load_event, load_dataset\nfrom trackml.score import score_event\nfrom trackml.randomize import shuffle_hits\n\nfrom sklearn.preprocessing import StandardScaler\nimport hdbscan as _hdbscan\nfrom scipy import stats\nfrom tqdm import tqdm\n\nimport time\n\nfrom sklearn.cluster.dbscan_ import dbscan\nfrom sklearn.preprocessing import StandardScaler\n\n\n# https://www.ellicium.com/python-multiprocessing-pool-process/\n# http://sebastianraschka.com/Articles/2014_multiprocessing.html\nfrom multiprocessing import Pool","execution_count":1,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5059897eee9a67d27957c255d0ac6ba4793b4b21"},"cell_type":"code","source":"\nimport os\nimport time\nimport hdbscan as _hdbscan\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n#------------------------------------------------------\n\ndef make_counts(labels):\n\n   _,reverse,count = np.unique(labels,return_counts=True,return_inverse=True)\n   counts = count[reverse]\n   counts[labels==0]=0\n\n   return counts\n\n\ndef one_loop(param):\n   # <todo> tune your parameters or design your own features here! \n\n   i,m, x,y,z, d,r, a, a_start,a_step = param\n   #print('\\r %3d  %+0.8f '%(i,da), end='', flush=True)\n\n   da = m*(a_start - (i*a_step))\n   aa = a + np.sign(z)*z*da \n   zr = z/r\n\n   X = StandardScaler().fit_transform(np.column_stack([ \n       aa, aa/zr, zr, 1/zr, aa/zr + 1/zr]))\n   _,l = dbscan(X, eps=0.0035, min_samples=1,)\n\n\n   return l\n\n\ndef do_dbscan_predict(df):\n\n   x  = df.x.values\n   y  = df.y.values\n   z  = df.z.values\n   r  = np.sqrt(x**2+y**2)\n   d  = np.sqrt(x**2+y**2+z**2)\n   a  = np.arctan2(y,x)\n\n   a_start,a_step,a_num = 0.00100,0.0000095,120\n   params  = [(i,m, x,y,z,d,r, a, a_start,a_step) for i in range(a_num) for m in [-1,1]]\n\n   if 1: \n       pool = Pool(processes=7)\n       ls   = pool.map( one_loop, params )\n\n   if 0:\n       ls = [ one_loop(param) for param in  params ]\n\n\n   ##------------------------------------------------\n\n   num_hits=len(df)\n   labels = np.zeros(num_hits,np.int32)\n   counts = np.zeros(num_hits,np.int32)\n   for l in ls:\n       c = make_counts(l)\n       idx = np.where((c-counts>0) & (c<20))[0]\n       labels[idx] = l[idx] + labels.max()\n       counts = make_counts(labels)\n\n   return labels\n\n\n# In[6]:\n\n\n## reference----------------------------------------------\ndef do_dbscan0_predict(df):\n    x = df.x.values\n    y = df.y.values\n    z = df.z.values\n    r = np.sqrt(x**2+y**2)\n    d = np.sqrt(x**2+y**2+z**2)\n\n    X = StandardScaler().fit_transform(np.column_stack([\n        x/d, y/d, z/r]))\n    _,labels = dbscan(X,\n                eps=0.0075,\n                min_samples=1,\n                algorithm='auto',\n                n_jobs=-1)\n\n    #labels = hdbscan(X, min_samples=1, min_cluster_size=5, cluster_selection_method='eom')\n\n    return labels\n\n\n# In[ ]:\n\n\n## reference----------------------------------------------\ndef do_dbscan0_predict(df):\n    x = df.x.values\n    y = df.y.values\n    z = df.z.values\n    r = np.sqrt(x**2+y**2)\n    d = np.sqrt(x**2+y**2+z**2)\n\n    X = StandardScaler().fit_transform(np.column_stack([\n        x/d, y/d, z/r]))\n    _,labels = dbscan(X,\n                eps=0.0075,\n                min_samples=1,\n                algorithm='auto',\n                n_jobs=-1)\n\n    #labels = hdbscan(X, min_samples=1, min_cluster_size=5, cluster_selection_method='eom')\n\n    return labels\n\n\n\n\n#########################################\n\ndef run_dbscan():\n\n    data_dir = '../input/train_100_events'\n\n    event_ids = [\n            '000001030',##\n            '000001025','000001026','000001027','000001028','000001029',\n    ]\n\n    sum=0\n    sum_score=0\n    for i,event_id in enumerate(event_ids):\n        particles = pd.read_csv(data_dir + '/event%s-particles.csv'%event_id)\n        hits  = pd.read_csv(data_dir + '/event%s-hits.csv'%event_id)\n        cells = pd.read_csv(data_dir + '/event%s-cells.csv'%event_id)\n        truth = pd.read_csv(data_dir + '/event%s-truth.csv'%event_id)\n\n        track_id = do_dbscan_predict(hits)\n\n        submission = pd.DataFrame(columns=['event_id', 'hit_id', 'track_id'],\n            data=np.column_stack(([int(event_id),]*len(hits), hits.hit_id.values, track_id))\n        ).astype(int)\n\n        score = score_event(truth, submission)\n        print('[%2d] score : %0.8f'%(i, score))\n        sum_score += score\n        sum += 1\n\n    print('--------------------------------------')\n    print(sum_score/sum)\n\n\n#########################################\n\n\ndef run_make_submission():\n\n    data_dir = '../input/test'\n\n\n    tic = t = time.time()\n    event_ids = [ '%09d'%i for i in range(0,125) ]  #(0,125)\n\n    if 1:\n        submissions = []\n        for i,event_id in enumerate(event_ids):\n            hits  = pd.read_csv(data_dir + '/event%s-hits.csv'%event_id)\n            cells = pd.read_csv(data_dir + '/event%s-cells.csv'%event_id)\n\n            track_id = do_dbscan_predict(hits)\n            #track_id = back_fit(track_id,hits)\n\n            toc =  time.time()\n            print('\\revent_id : %s  , %0.0f min'%(event_id, (toc-tic)/60))\n\n            # Prepare submission for an event\n            submission = pd.DataFrame(columns=['event_id', 'hit_id', 'track_id'],\n                data=np.column_stack(([event_id,]*len(hits), hits.hit_id.values, track_id))\n            ).astype(int)\n            submissions.append(submission)\n            submission.to_csv('./output/%s.csv.gz'%event_id,\n                                index=False, compression='gzip')\n\n            #------------------------------------------------------\n    if 1:\n\n        event_ids = [ '%09d'%i for i in range(0,125) ]  #(0,125)\n        submissions = []\n        for i,event_id in enumerate(event_ids):\n            submission  = pd.read_csv('./output/%s.csv.gz'%event_id, compression='gzip')\n            submissions.append(submission)\n\n\n        # Create submission file\n        submission = pd.concat(submissions, axis=0)\n        submission.to_csv('submission_dbscan.f24.csv.gz',\n                            index=False, compression='gzip')\n        print(len(submission))\n\n\n# main #################################################################\nif __name__ == '__main__':\n    print( '%s: calling main function ... ' % os.path.basename('__file__'))\n\n    run_dbscan()\n    run_make_submission()\n\n    print('\\nsucess!')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0a333cd4-351e-4274-aa7c-4cf8ab7fca1a","_uuid":"70ce31d93086e022159d6227f35c6488bf80eb22","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"anaconda-cloud":{},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}