{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Set Parameters\n**model_id** is the ID of the trained AutoML model\n\nThe model will only return models with a confidence above the **score threshold**. I set it really low just in case there are some labels that the model is particular unsure about.\n\n**gcp_service_account_json** is the path to the service account key. Service accounts allow you to authenticate with GCP using a JSON key (rather than typing in a password). I uploaded my service account key as a private dataset. Read more about setting one up at [https://cloud.google.com/iam/docs/understanding-service-accounts](https://cloud.google.com/iam/docs/understanding-service-accounts)\n\n**gcp_project_id**: in GCP, work is organized into projects.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_id = 'ICN2247013813647982800'\nscore_threshold = 0.000001\n\ngcp_service_account_json = '/kaggle/input/gcloudserviceaccountkey/kaggle-playground-170215-4ece6a076f22.json'\ngcp_project_id = 'kaggle-playground-170215'\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Install AutoML Python Package"},{"metadata":{"trusted":true},"cell_type":"code","source":"#AutoML package\n!pip install google-cloud-automl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import libraries\nDoing it after packages have been installed"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed\n        \nfrom google.cloud import automl_v1beta1\nfrom tqdm import tqdm\nimport operator\nimport os\nimport pandas as pd\nimport sys\nimport time\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define key functions\n**get_prediction()** is calls the modle endpoint and gets a prediction\n\n**make_int()** converts the returned prediction to an integer\n\n**process()** parses the response from the AutoML API\n\n**generated_predictions_with_pool_executor()** hits the API many times in parallel for faster processing\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"#generates predictions for all sirnas above the prediction threshold\ndef get_prediction(file_path, project_id, model_id):\n\n    name = 'projects/{}/locations/us-central1/models/{}'.format(project_id, model_id)\n    \n    with open(file_path, 'rb') as ff:\n        content = ff.read()\n        payload = {'image': {'image_bytes': content }}\n\n    params = {'score_threshold':str(score_threshold)}\n    request = prediction_client.predict(name, payload, params)\n        \n    return request\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#convert sirnas to integrater\ndef make_int(s):\n        try: \n            int(s)\n            return int(s) \n        except ValueError:\n            return 1109 #1109 is invalid so it's skipped over by the process function. That's why I set the label to 1109 for an error. \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process(i,df_sample_submission,project_id):\n    id_code = df_sample_submission.index[i]\n    if id_code in df_solution.index:\n        return None\n\n    exp_len = id_code.find('_')\n    experiment = id_code[0:exp_len]\n    plate = id_code[(exp_len+1):(exp_len+2)]\n    well = id_code[exp_len+3:]\n    pred_dict = {}\n\n    res = []\n\n    for site in range(1,3):\n        file_path = '../input/recursion_rgb_512/testrgb512/testRGB512/{}_{}_{}_s{}.png'.format(experiment,plate,well,site)\n      \n        try:\n            prediction_request = get_prediction(file_path, project_id,  model_id)\n        except Exception as e:\n            print('Something got wrong: ', e)\n            return None\n\n\n\n        for prediction in prediction_request.payload:\n            label = make_int(prediction.display_name)\n            if label <= 1108:\n                pred_dict[label] = float(prediction.classification.score)\n\n\n        sirna_prediction = max(pred_dict.items(), key=operator.itemgetter(1))[0] \n        confidence = pred_dict[sirna_prediction]\n\n        res.append({\n            'id_code': id_code, \n            'site': site, \n            'sirna_prediction': sirna_prediction, \n            'confidence': confidence\n        })\n\n    return res\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generated_predictions_with_pool_executor(max_workers,gcp_project_id):\n    results = []\n\n    df_sample_submission = pd.read_csv('../input/recursion-cellular-image-classification/sample_submission.csv',index_col=[0])\n    \n    with PoolExecutor(max_workers=max_workers) as executor:\n        futures_list = [executor.submit(process, i,df_sample_submission,gcp_project_id) for i in range(len(df_sample_submission))]\n        for f in tqdm(as_completed(futures_list), total=len(futures_list)):\n            results.append(f.result())\n\n    nb_escaped = 0\n    for r in results:\n        if r is None:\n            nb_escaped += 1\n            continue\n        for site in r:\n            df_solution.loc[site['id_code'], ['site{}_sirna'.format(site['site']),'site{}_confidence'.format(site['site'])]] = [site['sirna_prediction'], site['confidence']]\n\n    #df_solution.to_csv('./submissions/submission_{}.csv'.format(model_id))\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Send images to the inference API to generate predicted labels for each site\nThe output of this step is a csv file with the most likely sirna label. And the confidence score for that label.   "},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"solution_file_path = ',/submissions/submission_{}.csv'.format(model_id)\nif os.path.exists(solution_file_path):\n    df_solution = pd.read_csv(solution_file_path,index_col=[0])\nelse:\n    df_solution = pd.DataFrame(columns=['site1_sirna','site1_confidence','site2_sirna','site2_confidence'])\n    df_solution.index.name = 'id_code'\n\nprediction_client = automl_v1beta1.PredictionServiceClient.from_service_account_json(gcp_service_account_json)\n\ngenerated_predictions_with_pool_executor(20,gcp_project_id)\n#run the command a few more times just to pick up any rows that reported errors\ngenerated_predictions_with_pool_executor(5,gcp_project_id)\ngenerated_predictions_with_pool_executor(5,gcp_project_id)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pick the most confident sirna as final prediction\nLooks at the confidence score for each site and predicts the most likely sirna"},{"metadata":{"trusted":true},"cell_type":"code","source":"site1_wins = df_solution['site1_confidence'] >= df_solution['site2_confidence']\nsite2_wins = df_solution['site2_confidence'] > df_solution['site1_confidence']\n\ndf_solution.loc[site1_wins,'sirna'] = (df_solution[site1_wins])['site1_sirna']\ndf_solution.loc[site2_wins,'sirna'] = (df_solution[site2_wins])['site2_sirna']\ndf_solution['sirna'] = df_solution['sirna'].astype(int)\ndf_solution['sirna'].sort_index().to_csv('submit_{}.csv'.format(model_id),header=True)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}