Skip to content

Commit

Permalink
Add ipynb format check (#1439)
Browse files Browse the repository at this point in the history 
* Update test_qlib_from_source.yml

* add ipynb format check to workflow

* test ipynb CI

* modify nbqa check path

* add pylint flake8 mypy check to ipynb

* check ipynb with black and pylint

* reformat .ipynb files

* format line length

nbqa black . -l 120

* update nbqa .ipynb format CI

* format old ipynb files

* add nbconvert check to CI

* adjust CI order to avoid repeating download data
  • Loading branch information
@Cadenza-Li
Cadenza-Li authored Feb 21, 2023
1 parent 5eb5ac1 commit 76f2fb1
Show file tree
Hide file tree
Showing 6 changed files with 275 additions and 173 deletions.
11 changes: 11 additions & 0 deletions .github/workflows/test_qlib_from_source.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,11 @@ jobs:
run: |
mypy qlib --install-types --non-interactive || true
mypy qlib --verbose
- name: Check Qlib ipynb with nbqa
run: |
nbqa black . -l 120 --check --diff
nbqa pylint . --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136,W0719,W0104,W0404,C0412,W0611,C0410 --const-rgx='[a-z_][a-z0-9_]{2,30}$'
- name: Test data downloads
run: |
Expand All @@ -139,6 +144,12 @@ jobs:
brew unlink libomp
brew install libomp.rb
# Run after data downloads
- name: Check Qlib ipynb with nbconvert
run: |
# add more ipynb files in future
jupyter nbconvert --to notebook --execute examples/workflow_by_code.ipynb
- name: Test workflow by config (install from source)
run: |
python -m pip install numba
Expand Down
232 changes: 131 additions & 101 deletions examples/benchmarks/TRA/Reports.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,59 +25,65 @@
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib\n",
"sns.set(style='white')\n",
"matplotlib.rcParams['pdf.fonttype'] = 42\n",
"matplotlib.rcParams['ps.fonttype'] = 42\n",
"\n",
"sns.set(style=\"white\")\n",
"matplotlib.rcParams[\"pdf.fonttype\"] = 42\n",
"matplotlib.rcParams[\"ps.fonttype\"] = 42\n",
"\n",
"from tqdm.auto import tqdm\n",
"from joblib import Parallel, delayed\n",
"\n",
"\n",
"def func(x, N=80):\n",
" ret = x.ret.copy()\n",
" x = x.rank(pct=True)\n",
" x['ret'] = ret\n",
" x[\"ret\"] = ret\n",
" diff = x.score.sub(x.label)\n",
" r = x.nlargest(N, columns='score').ret.mean()\n",
" r -= x.nsmallest(N, columns='score').ret.mean()\n",
" return pd.Series({\n",
" 'MSE': diff.pow(2).mean(), \n",
" 'MAE': diff.abs().mean(), \n",
" 'IC': x.score.corr(x.label),\n",
" 'R': r\n",
" })\n",
" \n",
" r = x.nlargest(N, columns=\"score\").ret.mean()\n",
" r -= x.nsmallest(N, columns=\"score\").ret.mean()\n",
" return pd.Series(\n",
" {\n",
" \"MSE\": diff.pow(2).mean(),\n",
" \"MAE\": diff.abs().mean(),\n",
" \"IC\": x.score.corr(x.label),\n",
" \"R\": r,\n",
" }\n",
" )\n",
"\n",
"\n",
"ret = pd.read_pickle(\"data/ret.pkl\").clip(-0.1, 0.1)\n",
"\n",
"\n",
"def backtest(fname, **kwargs):\n",
" pred = pd.read_pickle(fname).loc['2018-09-21':'2020-06-30'] # test period\n",
" pred['ret'] = ret\n",
" pred = pd.read_pickle(fname).loc[\"2018-09-21\":\"2020-06-30\"] # test period\n",
" pred[\"ret\"] = ret\n",
" dates = pred.index.unique(level=0)\n",
" res = Parallel(n_jobs=-1)(delayed(func)(pred.loc[d], **kwargs) for d in dates)\n",
" res = {\n",
" dates[i]: res[i]\n",
" for i in range(len(dates))\n",
" }\n",
" res = {dates[i]: res[i] for i in range(len(dates))}\n",
" res = pd.DataFrame(res).T\n",
" r = res['R'].copy()\n",
" r = res[\"R\"].copy()\n",
" r.index = pd.to_datetime(r.index)\n",
" r = r.reindex(pd.date_range(r.index[0], r.index[-1])).fillna(0) # paper use 365 days\n",
" return {\n",
" 'MSE': res['MSE'].mean(),\n",
" 'MAE': res['MAE'].mean(),\n",
" 'IC': res['IC'].mean(),\n",
" 'ICIR': res['IC'].mean()/res['IC'].std(),\n",
" 'AR': r.mean()*365,\n",
" 'AV': r.std()*365**0.5,\n",
" 'SR': r.mean()/r.std()*365**0.5,\n",
" 'MDD': (r.cumsum().cummax() - r.cumsum()).max()\n",
" \"MSE\": res[\"MSE\"].mean(),\n",
" \"MAE\": res[\"MAE\"].mean(),\n",
" \"IC\": res[\"IC\"].mean(),\n",
" \"ICIR\": res[\"IC\"].mean() / res[\"IC\"].std(),\n",
" \"AR\": r.mean() * 365,\n",
" \"AV\": r.std() * 365**0.5,\n",
" \"SR\": r.mean() / r.std() * 365**0.5,\n",
" \"MDD\": (r.cumsum().cummax() - r.cumsum()).max(),\n",
" }, r\n",
"\n",
"\n",
"def fmt(x, p=3, scale=1, std=False):\n",
" _fmt = '{:.%df}'%p\n",
" _fmt = \"{:.%df}\" % p\n",
" string = _fmt.format((x.mean() if not isinstance(x, (float, np.floating)) else x) * scale)\n",
" if std and len(x) > 1:\n",
" string += ' ('+_fmt.format(x.std()*scale)+')'\n",
" string += \" (\" + _fmt.format(x.std() * scale) + \")\"\n",
" return string\n",
"\n",
"\n",
"def backtest_multi(files, **kwargs):\n",
" res = []\n",
" pnl = []\n",
Expand All @@ -88,14 +94,14 @@
" res = pd.DataFrame(res)\n",
" pnl = pd.concat(pnl, axis=1)\n",
" return {\n",
" 'MSE': fmt(res['MSE'], std=True),\n",
" 'MAE': fmt(res['MAE'], std=True),\n",
" 'IC': fmt(res['IC']),\n",
" 'ICIR': fmt(res['ICIR']),\n",
" 'AR': fmt(res['AR'], scale=100, p=1)+'%',\n",
" 'VR': fmt(res['AV'], scale=100, p=1)+'%',\n",
" 'SR': fmt(res['SR']),\n",
" 'MDD': fmt(res['MDD'], scale=100, p=1)+'%'\n",
" \"MSE\": fmt(res[\"MSE\"], std=True),\n",
" \"MAE\": fmt(res[\"MAE\"], std=True),\n",
" \"IC\": fmt(res[\"IC\"]),\n",
" \"ICIR\": fmt(res[\"ICIR\"]),\n",
" \"AR\": fmt(res[\"AR\"], scale=100, p=1) + \"%\",\n",
" \"VR\": fmt(res[\"AV\"], scale=100, p=1) + \"%\",\n",
" \"SR\": fmt(res[\"SR\"]),\n",
" \"MDD\": fmt(res[\"MDD\"], scale=100, p=1) + \"%\",\n",
" }, pnl"
]
},
Expand Down Expand Up @@ -124,16 +130,20 @@
"outputs": [],
"source": [
"exps = {\n",
" 'Linear': ['output/Linear/pred.pkl'],\n",
" 'LightGBM': ['output/GBDT/lr0.05_leaves128/pred.pkl'],\n",
" 'MLP': glob.glob('output/search/MLP/hs128_bs512_do0.3_lr0.001_seed*/pred.pkl'),\n",
" 'SFM': glob.glob('output/search/SFM/hs32_bs512_do0.5_lr0.001_seed*/pred.pkl'),\n",
" 'ALSTM': glob.glob('output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
" 'Trans.': glob.glob('output/search/Transformer/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
" 'ALSTM+TS':glob.glob('output/LSTM_Attn_TS/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
" 'Trans.+TS':glob.glob('output/Transformer_TS/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
" 'ALSTM+TRA(Ours)': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
" 'Trans.+TRA(Ours)': glob.glob('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb1.0_head4_hs64_bs512_do0.1_lr0.0005_seed*/pred.pkl')\n",
" \"Linear\": [\"output/Linear/pred.pkl\"],\n",
" \"LightGBM\": [\"output/GBDT/lr0.05_leaves128/pred.pkl\"],\n",
" \"MLP\": glob.glob(\"output/search/MLP/hs128_bs512_do0.3_lr0.001_seed*/pred.pkl\"),\n",
" \"SFM\": glob.glob(\"output/search/SFM/hs32_bs512_do0.5_lr0.001_seed*/pred.pkl\"),\n",
" \"ALSTM\": glob.glob(\"output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl\"),\n",
" \"Trans.\": glob.glob(\"output/search/Transformer/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl\"),\n",
" \"ALSTM+TS\": glob.glob(\"output/LSTM_Attn_TS/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl\"),\n",
" \"Trans.+TS\": glob.glob(\"output/Transformer_TS/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl\"),\n",
" \"ALSTM+TRA(Ours)\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
" \"Trans.+TRA(Ours)\": glob.glob(\n",
" \"output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb1.0_head4_hs64_bs512_do0.1_lr0.0005_seed*/pred.pkl\"\n",
" ),\n",
"}"
]
},
Expand All @@ -160,14 +170,8 @@
}
],
"source": [
"res = {\n",
" name: backtest_multi(exps[name])\n",
" for name in tqdm(exps)\n",
"}\n",
"report = pd.DataFrame({\n",
" k: v[0]\n",
" for k, v in res.items()\n",
"}).T"
"res = {name: backtest_multi(exps[name]) for name in tqdm(exps)}\n",
"report = pd.DataFrame({k: v[0] for k, v in res.items()}).T"
]
},
{
Expand Down Expand Up @@ -385,24 +389,40 @@
}
],
"source": [
"df = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed1000/pred.pkl')\n",
"code = 'SH600157'\n",
"date = '2018-09-28'\n",
"df = pd.read_pickle(\n",
" \"output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed1000/pred.pkl\"\n",
")\n",
"code = \"SH600157\"\n",
"date = \"2018-09-28\"\n",
"lookbackperiod = 50\n",
"\n",
"prob = df.iloc[:, -3:].loc(axis=0)[:, code].reset_index(level=1, drop=True).loc[date:].iloc[:lookbackperiod]\n",
"pred = df.loc[:,[\"score_0\",\"score_1\",\"score_2\",\"label\"]].loc(axis=0)[:, code].reset_index(level=1, drop=True).loc[date:].iloc[:lookbackperiod]\n",
"e_all = pred.iloc[:,:-1].sub(pred.iloc[:,-1], axis=0).pow(2)\n",
"pred = (\n",
" df.loc[:, [\"score_0\", \"score_1\", \"score_2\", \"label\"]]\n",
" .loc(axis=0)[:, code]\n",
" .reset_index(level=1, drop=True)\n",
" .loc[date:]\n",
" .iloc[:lookbackperiod]\n",
")\n",
"e_all = pred.iloc[:, :-1].sub(pred.iloc[:, -1], axis=0).pow(2)\n",
"e_all = e_all.sub(e_all.min(axis=1), axis=0)\n",
"e_all.columns = [r'$\\theta_%d$'%d for d in range(1, 4)]\n",
"e_all.columns = [r\"$\\theta_%d$\" % d for d in range(1, 4)]\n",
"prob = pd.Series(np.argmax(prob.values, axis=1), index=prob.index).rolling(7).mean().round()\n",
"\n",
"fig, axes = plt.subplots(1, 2, figsize=(7, 3))\n",
"e_all.plot(ax=axes[0], xlabel='', rot=30)\n",
"prob.plot(ax=axes[1], xlabel='', rot=30, color='red', linestyle='None', marker='^', markersize=5)\n",
"e_all.plot(ax=axes[0], xlabel=\"\", rot=30)\n",
"prob.plot(\n",
" ax=axes[1],\n",
" xlabel=\"\",\n",
" rot=30,\n",
" color=\"red\",\n",
" linestyle=\"None\",\n",
" marker=\"^\",\n",
" markersize=5,\n",
")\n",
"plt.yticks(np.array([0, 1, 2]), e_all.columns.values)\n",
"axes[0].set_ylabel('Predictor Loss')\n",
"axes[1].set_ylabel('Router Selection')\n",
"axes[0].set_ylabel(\"Predictor Loss\")\n",
"axes[1].set_ylabel(\"Router Selection\")\n",
"plt.tight_layout()\n",
"# plt.savefig('select.pdf', bbox_inches='tight')\n",
"plt.show()"
Expand All @@ -428,10 +448,18 @@
"outputs": [],
"source": [
"exps = {\n",
" 'Random': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcNONE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
" 'LR': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcLR_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
" 'TPE': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcTPE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
" 'LR+TPE': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl')\n",
" \"Random\": glob.glob(\n",
" \"output/search/LSTM_Attn_tra/K10_traHs16_traSrcNONE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
" \"LR\": glob.glob(\n",
" \"output/search/LSTM_Attn_tra/K10_traHs16_traSrcLR_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
" \"TPE\": glob.glob(\n",
" \"output/search/LSTM_Attn_tra/K10_traHs16_traSrcTPE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
" \"LR+TPE\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
"}"
]
},
Expand All @@ -456,14 +484,8 @@
}
],
"source": [
"res = {\n",
" name: backtest_multi(exps[name])\n",
" for name in tqdm(exps)\n",
"}\n",
"report = pd.DataFrame({\n",
" k: v[0]\n",
" for k, v in res.items()\n",
"}).T"
"res = {name: backtest_multi(exps[name]) for name in tqdm(exps)}\n",
"report = pd.DataFrame({k: v[0] for k, v in res.items()}).T"
]
},
{
Expand Down Expand Up @@ -597,18 +619,22 @@
}
],
"source": [
"a = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl')\n",
"b = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl')\n",
"a = pd.read_pickle(\n",
" \"output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl\"\n",
")\n",
"b = pd.read_pickle(\n",
" \"output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl\"\n",
")\n",
"a = a.iloc[:, -3:]\n",
"b = b.iloc[:, -3:]\n",
"b = np.eye(3)[b.values.argmax(axis=1)]\n",
"a = np.eye(3)[a.values.argmax(axis=1)]\n",
"\n",
"res = pd.DataFrame({\n",
" 'with OT': b.sum(axis=0) / b.sum(),\n",
" 'without OT': a.sum(axis=0)/ a.sum() \n",
"},index=[r'$\\theta_1$',r'$\\theta_2$',r'$\\theta_3$'])\n",
"res.plot.bar(rot=30, figsize=(5, 4), color=['b', 'g'])\n",
"res = pd.DataFrame(\n",
" {\"with OT\": b.sum(axis=0) / b.sum(), \"without OT\": a.sum(axis=0) / a.sum()},\n",
" index=[r\"$\\theta_1$\", r\"$\\theta_2$\", r\"$\\theta_3$\"],\n",
")\n",
"res.plot.bar(rot=30, figsize=(5, 4), color=[\"b\", \"g\"])\n",
"del a, b"
]
},
Expand All @@ -633,11 +659,19 @@
"outputs": [],
"source": [
"exps = {\n",
" 'K=1': glob.glob('output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/info.json'),\n",
" 'K=3': glob.glob('output/search/finetune/LSTM_Attn_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
" 'K=5': glob.glob('output/search/finetune/LSTM_Attn_tra/K5_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
" 'K=10': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
" 'K=20': glob.glob('output/search/finetune/LSTM_Attn_tra/K20_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json')\n",
" \"K=1\": glob.glob(\"output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/info.json\"),\n",
" \"K=3\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json\"\n",
" ),\n",
" \"K=5\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K5_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json\"\n",
" ),\n",
" \"K=10\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json\"\n",
" ),\n",
" \"K=20\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K20_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json\"\n",
" ),\n",
"}"
]
},
Expand All @@ -649,16 +683,11 @@
"source": [
"report = dict()\n",
"for k, v in exps.items():\n",
" \n",
" tmp = dict()\n",
" for fname in v:\n",
" with open(fname) as f:\n",
" info = json.load(f)\n",
" tmp[fname] = (\n",
" {\n",
" \"IC\":info[\"metric\"][\"IC\"],\n",
" \"MSE\":info[\"metric\"][\"MSE\"]\n",
" })\n",
" tmp[fname] = {\"IC\": info[\"metric\"][\"IC\"], \"MSE\": info[\"metric\"][\"MSE\"]}\n",
" tmp = pd.DataFrame(tmp).T\n",
" report[k] = tmp.mean()\n",
"report = pd.DataFrame(report).T"
Expand All @@ -681,13 +710,14 @@
}
],
"source": [
"fig, axes = plt.subplots(1, 2, figsize=(6,3)); axes = axes.flatten()\n",
"report['IC'].plot.bar(rot=30, ax=axes[0])\n",
"fig, axes = plt.subplots(1, 2, figsize=(6, 3))\n",
"axes = axes.flatten()\n",
"report[\"IC\"].plot.bar(rot=30, ax=axes[0])\n",
"axes[0].set_ylim(0.045, 0.062)\n",
"axes[0].set_title('IC performance')\n",
"report['MSE'].astype(float).plot.bar(rot=30, ax=axes[1], color='green')\n",
"axes[0].set_title(\"IC performance\")\n",
"report[\"MSE\"].astype(float).plot.bar(rot=30, ax=axes[1], color=\"green\")\n",
"axes[1].set_ylim(0.155, 0.1585)\n",
"axes[1].set_title('MSE performance')\n",
"axes[1].set_title(\"MSE performance\")\n",
"plt.tight_layout()\n",
"# plt.savefig('sensitivity.pdf')"
]
Expand Down
Loading

0 comments on commit 76f2fb1

Please sign in to comment.