{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from sklearn import *\nimport numpy as np\nimport pandas as pd\nimport random\nimport glob\n\nsdir = '/kaggle/input/AI4Code/'\ntrain = glob.glob(sdir+'/train/**') #139256\ntest = glob.glob(sdir+'/test/**') #4 / 20K code comp\nsub = pd.read_csv(sdir+'sample_submission.csv') #id, cell_order","metadata":{"execution":{"iopub.status.busy":"2022-08-07T10:48:06.733632Z","iopub.execute_input":"2022-08-07T10:48:06.734160Z","iopub.status.idle":"2022-08-07T10:48:11.139973Z","shell.execute_reply.started":"2022-08-07T10:48:06.734066Z","shell.execute_reply":"2022-08-07T10:48:11.139243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef getDF(files):\n    dfs = []\n    for f in files:\n        d = eval(open(f, 'r').read())\n        df = pd.DataFrame([[c, r, d['cell_type'][c], d['source'][c]] for r, c in enumerate(d['cell_type'])], columns=['cell_id','rank','cell_type','source'])\n        df['id'] = f.split('/')[-1].split('.')[0]\n        dfs.append(df)\n    return pd.concat(dfs)\n\ntrain = getDF(train)\ntest = getDF(test)\nprint(train.shape, test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T10:48:43.707945Z","iopub.execute_input":"2022-08-07T10:48:43.708225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cleanstr(s):\n    #s = str(s).encode('unicode-escape').decode('ascii')\n    s = s.replace('<br\\\\/>', ' ')\n    s = s.replace('\\\\n', ' ')\n    s = s.replace('\\\\/', ' ')\n    s = s.replace('\\\\t', ' ')\n    s = s.replace('#', ' ')\n    s = s.replace('*', ' ')\n    s = s.replace(',', ' ')\n    s = s.replace('.', ' ')\n    s = s.replace('](', ' ')\n    s = s.replace('  ', ' ')\n    return s\n\nprint(train['cell_type'].value_counts())\ndf = train.drop_duplicates(subset=['id'], keep='last')\ndf = df[['id','rank']].rename(columns={'rank':'total'})\ntrain = pd.merge(train, df, how='left', on='id')\ntrain['target'] = train['rank']/train['total']\ntrain = train[train.cell_type=='markdown']\ntrain['source'] = train['source'].map(lambda x: cleanstr(x))\n\nprint(test['cell_type'].value_counts())\ndf = test.drop_duplicates(subset=['id'], keep='last')\ndf = df[['id','rank']].rename(columns={'rank':'total'})\ntest = pd.merge(test, df, how='left', on='id')\ntest['target'] = test['rank']/test['total']\ntest['source'] = test['source'].map(lambda x: cleanstr(x)) #impacts code but ok","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef fdeffec(c, s, exp = {}):\n    it = {'a':10,\n        'b':0.5,\n        'c':0.5,\n        'd':0.9,\n        'e':1e-10}\n    for i in range(len(c)):\n        words = set([w for w in str(c[i]).lower().split(' ')])\n        for w in words:\n            try:\n                exp[w]['b'] = exp[w]['b'][1:] + [s[i]]\n                exp[w]['c'] += 1\n                exp[w]['d'] = exp[w]['d'][1:] + [(exp[w]['d'][it['a']-1] + (s[i] * it['d']))/2]\n                exp[w]['e'] += s[i]\n            except:\n                m = [0. for m_ in range(it['a'])]\n                exp[w] = {}\n                exp[w]['b'] = m[1:] + [s[i]]\n                exp[w]['c'] = 1\n                exp[w]['d'] = m[1:] + [s[i] * it['d'] / 2]\n                exp[w]['e'] = s[i]\n                \n    for w in exp:\n        exp[w]['e'] /= exp[w]['c'] + it['e']\n        exp[w]['c'] /= len(c) * it['c']\n\n    return exp\n\nexp = fdeffec(train['source'].values, train['target'].values)\n#exp = fdeffec(test[test.cell_type=='markdown']['source'].values, np.zeros(len(test[test.cell_type=='markdown'])), exp)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef features(df):\n    df['len'] = df['source'].map(len)\n    df['wlen'] = df['source'].map(lambda x: len(str(x).split(' ')))\n\n    df['b_mean'] = df['source'].map(lambda x: np.mean([np.mean(exp[w]['b']) if w in exp else 0 for w in str(x).lower().split(' ')]))\n    df['c_mean'] = df['source'].map(lambda x: np.mean([exp[w]['c'] if w in exp else 0 for w in str(x).lower().split(' ')]))\n    df['d_mean'] = df['source'].map(lambda x: np.mean([np.mean(exp[w]['d']) if w in exp else 0 for w in str(x).lower().split(' ')]))\n    df['e_mean'] = df['source'].map(lambda x: np.mean([exp[w]['e'] if w in exp else 0.5 for w in str(x).lower().split(' ')]))\n    return df\n\ntrain = features(train)\ntest = pd.concat((features(test[test.cell_type=='markdown']), test[test.cell_type=='code']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = [c for c in train if c not in ['cell_id', 'rank', 'cell_type', 'source', 'id', 'target']]\nx1, x2, y1, y2 = model_selection.train_test_split(train[col], train.target, test_size=0.2, random_state=2, stratify=train[['total']], shuffle=True)\n\nmodel = ensemble.ExtraTreesRegressor(n_estimators=2000, max_depth=14, n_jobs=-1, random_state=2)\nmodel.fit(x1, y1)\npreds = model.predict(x2)\nscore = metrics.r2_score(y2, preds)\nprint(score)\nmodel.fit(train[col], train.target)\n\ntest['target'] = model.predict(test.fillna(0)[col])\ndf = test[test.cell_type=='code']\ndf = df.groupby(by=['id'])['rank'].count().reset_index().rename(columns={'rank':'code_count'})\ntest = pd.merge(test, df, how='left', on='id')\ntest['target'] = test.apply(lambda r: r.target if r.cell_type=='markdown' else r['rank'] / r.code_count, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = []\nfor f in test['id'].unique():\n    m = list(test[test['id']==f].sort_values('target')['cell_id'].values)\n    sub.append([f, ' '.join(m)])\nsub = pd.DataFrame(sub, columns=['id', 'cell_order'])\nsub.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}