{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Load libraries\nimport os, warnings\nwarnings.filterwarnings(\"ignore\")\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.colors as mpl\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import init_notebook_mode\nfrom scipy.special import boxcox1p\nfrom scipy.stats import boxcox_normmax\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold\nfrom sklearn.metrics import accuracy_score, roc_curve, roc_auc_score, auc, classification_report\n\n# Load data\ntrain = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv', index_col=0)\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv', index_col=0)\n\n\nprint('Train Shape: {}\\nMissing Data: {}\\nDuplicates: {}\\n'\\\n      .format(train.shape, train.isna().sum().sum(), train.duplicated().sum()))\nprint('Test Shape: {}\\nMissing Data: {}\\nDuplicates: {}\\n'\\\n      .format(test.shape, test.isna().sum().sum(), test.duplicated().sum()))\ntrain_d=train.drop_duplicates() \nprint('Dropping Duplicates\\nNew Train Shape: {}'.format(train_d.shape))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-01T07:24:50.518031Z","iopub.execute_input":"2022-08-01T07:24:50.518550Z","iopub.status.idle":"2022-08-01T07:24:52.133145Z","shell.execute_reply.started":"2022-08-01T07:24:50.518449Z","shell.execute_reply":"2022-08-01T07:24:52.131743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Quick insights:\n1) there are 26570 rows and 25 columns with 20273 values missing and no duplicates","metadata":{}},{"cell_type":"code","source":"# total number of rows with missing values\ntrain.isnull().any(axis=1).sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:52.135341Z","iopub.execute_input":"2022-08-01T07:24:52.136557Z","iopub.status.idle":"2022-08-01T07:24:52.152999Z","shell.execute_reply.started":"2022-08-01T07:24:52.136504Z","shell.execute_reply":"2022-08-01T07:24:52.151768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the number of missing data points per column\nmissing_values_count = train.isnull().sum()\n\n# look at the # of missing points in the first ten columns\nmissing_values_count","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:52.154845Z","iopub.execute_input":"2022-08-01T07:24:52.155555Z","iopub.status.idle":"2022-08-01T07:24:52.173276Z","shell.execute_reply.started":"2022-08-01T07:24:52.155510Z","shell.execute_reply":"2022-08-01T07:24:52.172009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### There are large number of missing values, lets visualize them","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.colors\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import init_notebook_mode\nimport seaborn as sns\nplt.figure(figsize=(20,8))\nsns.heatmap(train.isnull(), yticklabels=False, cbar=False, cmap='crest')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:52.176047Z","iopub.execute_input":"2022-08-01T07:24:52.177025Z","iopub.status.idle":"2022-08-01T07:24:53.140588Z","shell.execute_reply.started":"2022-08-01T07:24:52.176977Z","shell.execute_reply":"2022-08-01T07:24:53.139358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### As we can see above large number of missing values. Need to find sophesticated system to handle them.","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:53.142362Z","iopub.execute_input":"2022-08-01T07:24:53.142726Z","iopub.status.idle":"2022-08-01T07:24:53.181153Z","shell.execute_reply.started":"2022-08-01T07:24:53.142692Z","shell.execute_reply":"2022-08-01T07:24:53.179656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp=dict(layout=go.Layout(font=dict(family=\"Franklin Gothic\", size=12), \n                           height=500, width=1000))\ntarget=train.failure.value_counts(normalize=True)\ntarget.rename(index={1:'Failed',0:'Passed'},inplace=True)\npal, color=['#016CC9','#DEB078'], ['#8DBAE2','#EDD3B3']\nfig=go.Figure()\nfig.add_trace(go.Pie(labels=target.index, values=target*100, hole=.45, \n                     showlegend=True,sort=False, \n                     marker=dict(colors=color,line=dict(color=pal,width=2.5)),\n                     hovertemplate = \"%{label} Accounts: %{value:.2f}%<extra></extra>\"))\nfig.update_layout(template=temp, title='Target Distribution', \n                  legend=dict(traceorder='reversed',y=1.05,x=0),\n                  uniformtext_minsize=15, uniformtext_mode='hide',width=700)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:53.183381Z","iopub.execute_input":"2022-08-01T07:24:53.183891Z","iopub.status.idle":"2022-08-01T07:24:53.242212Z","shell.execute_reply.started":"2022-08-01T07:24:53.183842Z","shell.execute_reply":"2022-08-01T07:24:53.241008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is only 21% of data showing failed status zero. Do we need to use class weights during model development? Need further investigation.","metadata":{}},{"cell_type":"code","source":"cor=train.corr()    \ncor.style.background_gradient(cmap='viridis')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:53.243947Z","iopub.execute_input":"2022-08-01T07:24:53.244361Z","iopub.status.idle":"2022-08-01T07:24:53.410314Z","shell.execute_reply.started":"2022-08-01T07:24:53.244326Z","shell.execute_reply":"2022-08-01T07:24:53.409167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature correlation with target\nimport matplotlib.pyplot as plt\nimport matplotlib.colors\n\ncorr=train.corr()\ncorr=corr['failure'].sort_values(ascending=False)[1:-1]\npal=sns.color_palette(\"Reds_r\",135).as_hex()\nrgb=['rgba'+str(matplotlib.colors.to_rgba(i,0.7)) for i in pal]\nfig = go.Figure()\nfig.add_trace(go.Bar(x=corr[corr>=0], y=corr[corr>=0].index, \n                     marker_color=rgb, orientation='h', \n                     marker_line=dict(color=pal,width=2), name='',\n                     hovertemplate='%{y} correlation with target: %{x:.3f}',\n                     showlegend=False))\npal=sns.color_palette(\"Blues\",100).as_hex()\nrgb=['rgba'+str(matplotlib.colors.to_rgba(i,0.7)) for i in pal]\nfig.add_trace(go.Bar(x=corr[corr<0], y=corr[corr<0].index, \n                     marker_color=rgb[25:], orientation='h', \n                     marker_line=dict(color=pal[25:],width=2), name='',\n                     hovertemplate='%{y} correlation with target: %{x:.3f}',\n                     showlegend=False))\nfig.update_layout(template=temp,title=\"Feature Correlations with Target\",\n                  xaxis_title=\"Correlation\", margin=dict(l=150),\n                  height=3000, width=700, hovermode='closest')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:53.412133Z","iopub.execute_input":"2022-08-01T07:24:53.412507Z","iopub.status.idle":"2022-08-01T07:24:53.518774Z","shell.execute_reply.started":"2022-08-01T07:24:53.412475Z","shell.execute_reply":"2022-08-01T07:24:53.517886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Quick insights:\n1) some of the features are negatively correlated. Interesting, need further investigation","metadata":{}},{"cell_type":"code","source":"cols=[col for col in train.columns if (col.startswith(('measurement_')))]\nplot_df=train[cols]\nfig, ax = plt.subplots(1,4, figsize=(16,5))\nfig.suptitle('Relationships between measurements',fontsize=16)\nax[0].hexbin(x='measurement_10', y='measurement_11', data=plot_df, bins='log', gridsize=40, cmap='coolwarm')\nax[0].set(xlabel='measurement_10',ylabel='measurement_11')\nax[0].text(1, 4, 'Correlation: {:.2f}'.format(plot_df[['measurement_10','measurement_11']].corr().iloc[1,0]), \n           ha=\"center\", va=\"center\",bbox=dict(boxstyle=\"round,pad=0.3\",fc=\"white\"))\nax[1].hexbin(x='measurement_13', y='measurement_10', data=plot_df, bins='log', gridsize=40, cmap='coolwarm')\nax[1].set(xlabel='measurement_13',ylabel='measurement_10')\nax[1].text(0.3, 4.2, 'Correlation: {:.2f}'.format(plot_df[['measurement_13','measurement_10']].corr().iloc[1,0]),\n           ha=\"center\", va=\"center\",bbox=dict(boxstyle=\"round,pad=0.3\",fc=\"white\"))\nax[2].hexbin(x='measurement_14', y='measurement_15', data=plot_df, bins='log', gridsize=40, cmap='coolwarm')\nax[2].set(xlabel='measurement_16',ylabel='measurement_17')\nax[2].text(2.15, 1.95, 'Correlation: {:.2f}'.format(plot_df[['measurement_14','measurement_15']].corr().iloc[1,0]), \n           ha=\"center\", va=\"center\",bbox=dict(boxstyle=\"round,pad=0.3\",fc=\"white\"))\nax[3].hexbin(x='measurement_8', y='measurement_9', data=plot_df, bins='log', gridsize=40, cmap='coolwarm')\nax[3].set(xlabel='measurement_8',ylabel='measurement_9')\nax[3].text(1.1, 5.9, 'Correlation: {:.2f}'.format(plot_df[['measurement_8','measurement_9']].corr().iloc[1,0]), \n           ha=\"center\", va=\"center\",bbox=dict(boxstyle=\"round,pad=0.3\",fc=\"white\"))\nfor i in range(4):\n    ax[i].tick_params(left=False,bottom=False)\nsns.despine()\nplt.tight_layout(rect=[0, 0, 1, 0.99])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:53.520136Z","iopub.execute_input":"2022-08-01T07:24:53.520493Z","iopub.status.idle":"2022-08-01T07:24:54.652470Z","shell.execute_reply.started":"2022-08-01T07:24:53.520461Z","shell.execute_reply":"2022-08-01T07:24:54.651385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_df = train.groupby(['attribute_2','measurement_0'])['failure'].value_counts(normalize=True)\nplot_df = plot_df.mul(100).rename('Percent').reset_index()\nfig = px.bar(plot_df, x=\"measurement_0\", y=\"Percent\", color=\"failure\", barmode=\"group\",\n            text='Percent', opacity=.75, facet_col=\"attribute_2\", category_orders={'failure': ['Yes', 'No']},\n            color_discrete_map={'Yes': '#C02B34','No': '#CDBBA7'}) \nfig.update_traces(texttemplate='%{text:.3s}%', textposition='outside',\n                  marker_line=dict(width=1, color='#28221D'),  width=.4)\nfig.update_layout(title_text='Failure Rates by measurement_0 and attribute_2', yaxis_ticksuffix = '%',\n                  paper_bgcolor='#F4F2F0', plot_bgcolor='#F4F2F0',font_color='#28221D',\n                  height=500, xaxis=dict(tickangle=30))\nfig.update_xaxes(showticklabels=True,tickangle=30,col=2)\nfig.update_yaxes(title = \"\", zeroline=True, zerolinewidth=1, zerolinecolor='#28221D')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:54.655587Z","iopub.execute_input":"2022-08-01T07:24:54.656603Z","iopub.status.idle":"2022-08-01T07:24:55.026813Z","shell.execute_reply.started":"2022-08-01T07:24:54.656563Z","shell.execute_reply":"2022-08-01T07:24:55.025892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_df = train.groupby(['attribute_1','measurement_0'])['failure'].value_counts(normalize=True)\nplot_df = plot_df.mul(100).rename('Percent').reset_index()\nfig = px.bar(plot_df, x=\"measurement_0\", y=\"Percent\", color=\"failure\", barmode=\"group\",\n            text='Percent', opacity=.75, facet_col=\"attribute_1\", category_orders={'failure': ['Yes', 'No']},\n            color_discrete_map={'Yes': '#C02B34','No': '#CDBBA7'}) \nfig.update_traces(texttemplate='%{text:.3s}%', textposition='outside',\n                  marker_line=dict(width=1, color='#28221D'),  width=.4)\nfig.update_layout(title_text='Failure Rates by measurement_0 and attribute_1', yaxis_ticksuffix = '%',\n                  paper_bgcolor='#F4F2F0', plot_bgcolor='#F4F2F0',font_color='#28221D',\n                  height=500, xaxis=dict(tickangle=30))\nfig.update_xaxes(showticklabels=True,tickangle=30,col=2)\nfig.update_yaxes(title = \"\", zeroline=True, zerolinewidth=1, zerolinecolor='#28221D')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:55.028342Z","iopub.execute_input":"2022-08-01T07:24:55.029360Z","iopub.status.idle":"2022-08-01T07:24:55.170474Z","shell.execute_reply.started":"2022-08-01T07:24:55.029321Z","shell.execute_reply":"2022-08-01T07:24:55.169160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_df=train.sort_values(by=\"failure\")\nfig=px.histogram(plot_df, x='loading', color='failure', \n                 opacity=0.8, histnorm='density', barmode='overlay', marginal='box',\n                 color_discrete_map={'Yes': '#C02B34','No': '#CDBBA7'})\nfig.update_layout(title_text='Distribution of loading by failure Status',\n                  xaxis_title='loading value', yaxis_title='Density',font_color='#28221D',\n                  paper_bgcolor='#F4F2F0', plot_bgcolor='#F4F2F0', legend_traceorder='reversed')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:55.171878Z","iopub.execute_input":"2022-08-01T07:24:55.172609Z","iopub.status.idle":"2022-08-01T07:24:55.307593Z","shell.execute_reply.started":"2022-08-01T07:24:55.172561Z","shell.execute_reply":"2022-08-01T07:24:55.306334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_df=train.sort_values(by=\"attribute_1\")\nfig=px.histogram(plot_df, x='loading', color='failure', \n                 opacity=0.8, histnorm='density', barmode='overlay', marginal='box',\n                 color_discrete_map={'Yes': '#C02B34','No': '#CDBBA7'})\nfig.update_layout(title_text='Distribution of attribute_1 by failure Status',\n                  xaxis_title='attribute_1 value', yaxis_title='Density',font_color='#28221D',\n                  paper_bgcolor='#F4F2F0', plot_bgcolor='#F4F2F0', legend_traceorder='reversed')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:55.309371Z","iopub.execute_input":"2022-08-01T07:24:55.310162Z","iopub.status.idle":"2022-08-01T07:24:55.439995Z","shell.execute_reply.started":"2022-08-01T07:24:55.310113Z","shell.execute_reply":"2022-08-01T07:24:55.439059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Similar plots can be generated for rest of the columns. As we can see above, attribute_1 has more density for failure status zero.","metadata":{}},{"cell_type":"code","source":"# Is there a relationship between failure status and the other features in the data set?\n# Let's use the chi-square test of independence. \n# The null hypothesis for this test is that the variables are independent,there is no association between Attrition and the variable being tested, while the alternative hypothesis is that that there is a relationship.\n#cat_cols=train.select_dtypes(include=\"category\").columns.tolist() \nimport scipy\nfrom scipy.stats import chi2_contingency \nchi_cols=[col for col in train.columns if (col.startswith(('attribute_', 'load', 'measu')))]\nchi_statistic=[]\np_val=[]\nvars_rm=[]\n\nfor i in train[chi_cols]:\n    observed=pd.crosstab(index=train[\"failure\"], columns=train[i])\n    stat, p, dof, expected=chi2_contingency(observed)\n    chi_statistic.append(stat)\n    p_val.append(p)\n    if p >= 0.05:\n        print(\"failure and {} are independent (p-value = {:.2f}).\\n\".format(i,p))\n        vars_rm.append(i)\n\nchi_df = pd.DataFrame()\nchi_df[\"Variable\"] = chi_cols\nchi_df[\"Chi_Statistic\"] = chi_statistic\nchi_df[\"P_value\"] = p_val\nchi_df=chi_df[chi_df.P_value<0.05].sort_values(\"P_value\", ascending=True)\ndisplay(chi_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:24:55.441200Z","iopub.execute_input":"2022-08-01T07:24:55.442002Z","iopub.status.idle":"2022-08-01T07:25:13.694083Z","shell.execute_reply.started":"2022-08-01T07:24:55.441965Z","shell.execute_reply":"2022-08-01T07:25:13.692701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Quick insights:\nBased on the chi square test results, factors that control the failure status is loading, attribute_0,attribute_1 and attribute_2. Interestingly failture status is independent of all the measuremnt columns. \nShould we drop these columns or not? further investigation is needed.","metadata":{}},{"cell_type":"code","source":"# set the max columns to none\npd.set_option('display.max_columns', None)\ntrain.tail()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:25:13.695333Z","iopub.execute_input":"2022-08-01T07:25:13.695653Z","iopub.status.idle":"2022-08-01T07:25:13.728931Z","shell.execute_reply.started":"2022-08-01T07:25:13.695625Z","shell.execute_reply":"2022-08-01T07:25:13.727614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MultiColumnLabelEncoder:\n    def __init__(self,columns = None):\n        self.columns = columns # array of column names to encode\n\n    def fit(self,X,y=None):\n        return self # not relevant here\n\n    def transform(self,X):\n        '''\n        Transforms columns of X specified in self.columns using\n        LabelEncoder(). If no columns specified, transforms all\n        columns in X.\n        '''\n        output = X.copy()\n        if self.columns is not None:\n            for col in self.columns:\n                output[col] = LabelEncoder().fit_transform(output[col])\n        else:\n            for colname,col in output.iteritems():\n                output[colname] = LabelEncoder().fit_transform(col)\n        return output\n\n    def fit_transform(self,X,y=None):\n        return self.fit(X,y).transform(X)\ntrain=MultiColumnLabelEncoder(columns = ['product_code','attribute_0','attribute_1']).fit_transform(train)\ntest=MultiColumnLabelEncoder(columns = ['product_code','attribute_0','attribute_1']).transform(test)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:25:13.730776Z","iopub.execute_input":"2022-08-01T07:25:13.731247Z","iopub.status.idle":"2022-08-01T07:25:13.816059Z","shell.execute_reply.started":"2022-08-01T07:25:13.731203Z","shell.execute_reply":"2022-08-01T07:25:13.814258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.fillna(test.mean())\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:25:13.817536Z","iopub.execute_input":"2022-08-01T07:25:13.817906Z","iopub.status.idle":"2022-08-01T07:25:13.863936Z","shell.execute_reply.started":"2022-08-01T07:25:13.817871Z","shell.execute_reply":"2022-08-01T07:25:13.862477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn import preprocessing\nfrom sklearn.metrics import mean_absolute_error, roc_auc_score\nfrom tqdm import tqdm\nfrom sklearn.ensemble import RandomForestClassifier\nimport random\nSEED = 2001\nN_FOLDS =5\ntrain_noNaN = train.dropna()\nauc_score = []\n\ncolumns = [col for col in train_noNaN.columns if col not in ['failure'] ]\n\n\ntargets = train_noNaN['failure'].values\n\nkf = StratifiedKFold(n_splits = N_FOLDS, shuffle=True, random_state=20)    \n        \noof = np.zeros((train_noNaN.shape[0],))\ntest_preds = 0\n\nfor f, (train_idx, val_idx) in tqdm(enumerate(kf.split(train_noNaN, targets))):\n        df_train, df_val = train_noNaN.iloc[train_idx][columns], train_noNaN.iloc[val_idx][columns]\n        train_target, val_target = targets[train_idx], targets[val_idx]\n        \n        model = RandomForestClassifier(\n            n_estimators=100,\n            max_depth=10,\n            random_state=SEED,\n            n_jobs=-1\n        )\n        \n        model.fit(df_train[columns], train_target)\n        \n        oof_tmp = model.predict_proba(df_val[columns])[:,1]\n        test_tmp = model.predict_proba(test[columns])[:,1]   \n        \n        oof[val_idx] = oof_tmp\n        test_preds += test_tmp/N_FOLDS\n        auc = roc_auc_score(val_target, oof_tmp)\n        auc_score.append(auc)\n        print(f'FOLD: {f} AUC: {auc} Mean AUC: {np.mean(auc_score)}')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:25:13.865740Z","iopub.execute_input":"2022-08-01T07:25:13.866163Z","iopub.status.idle":"2022-08-01T07:25:20.485259Z","shell.execute_reply.started":"2022-08-01T07:25:13.866125Z","shell.execute_reply":"2022-08-01T07:25:20.484047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## work in progress","metadata":{"execution":{"iopub.status.busy":"2022-08-01T07:25:20.486918Z","iopub.execute_input":"2022-08-01T07:25:20.487404Z","iopub.status.idle":"2022-08-01T07:25:20.492886Z","shell.execute_reply.started":"2022-08-01T07:25:20.487359Z","shell.execute_reply":"2022-08-01T07:25:20.491879Z"},"trusted":true},"execution_count":null,"outputs":[]}]}