diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5bd3fe8f..a8d84435 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -124,3 +124,19 @@ jobs: mkdir -p $LAZYLLM_HOME source ~/ENV/env.sh python -m pytest --lf --last-failed-no-failures=all --durations=0 --reruns=2 -v tests/charge_tests + + DocCheck: + runs-on: tps_sco_nv + needs: [ Clone ] + steps: + - name: RunTests + run: | + cd ${{ env.CI_PATH }} + pip install -r tests/requirements.txt + export PYTHONPATH=$PWD:$PYTHONPATH + export LAZYLLM_DATA_PATH=/mnt/lustre/share_data/lazyllm/data/ + export LAZYLLM_MODEL_PATH=/mnt/lustre/share_data/lazyllm/models + export LAZYLLM_HOME="${{ env.CI_PATH }}/${{ github.run_id }}-${{ github.job }}" + mkdir -p $LAZYLLM_HOME + source ~/ENV/env.sh + python -m pytest --lf --last-failed-no-failures=all --durations=0 --reruns=2 -v tests/doc_check diff --git a/tests/doc_check/doc_api_check.py b/tests/doc_check/doc_api_check.py new file mode 100644 index 00000000..0b578a93 --- /dev/null +++ b/tests/doc_check/doc_api_check.py @@ -0,0 +1,94 @@ +import pytest +import re +import inspect +import lazyllm +from typing import Callable +import warnings + + +def class_should_check(cls, module): + if not cls.__name__[0].isupper() or cls.__module__ != module.__name__: + return False + if cls.__module__ != module.__name__: + return False + all_methods = inspect.getmembers(cls, predicate=inspect.isfunction) + custom_methods = [name for name, func in all_methods if not name.startswith('_')] + return len(custom_methods) > 0 + + +def get_sub_classes(module): + clsmembers = inspect.getmembers(module, inspect.isclass) + classes = set([ele[1] for ele in clsmembers if class_should_check(ele[1], module)]) + for name, sub_module in inspect.getmembers(module, inspect.ismodule): + if sub_module.__name__.startswith(module.__name__): + classes.update(get_sub_classes(sub_module)) + return classes + + +def is_method_overridden(cls, method: Callable): + method_name = method.__name__ + for base in cls.__bases__: + if hasattr(base, method_name): + base_method = getattr(base, method_name) + current_method = getattr(cls, method_name) + if current_method != base_method: + return True + return False + + +def do_check_method(cls, func: Callable): + # As type is always missing in code signature and default value is not universal, + # Also Keyword argument is not universal. So we just check args parameter name + arg_spec = inspect.getfullargspec(func) + real_parms = arg_spec.args + arg_spec.kwonlyargs + real_vars = [arg_spec.varargs, arg_spec.varkw] + if real_parms[0] in ['self', 'cls']: + real_parms = real_parms[1:] + real_parms = set(real_parms) + if func.__name__ == '__init__': + doc = cls.__doc__ + else: + doc = func.__doc__ + if doc is not None: + seg_pattern = r"Args:\s*(.*?)\n\s*\n" + match = re.search(seg_pattern, doc, re.DOTALL) + doc_parms = [] + if match: + args_pattern = r"^\s*(\w+)\s*(?:\(|:)" + doc_parms = re.findall(args_pattern, match.group(1), re.MULTILINE) + for doc_param in doc_parms: + if doc_param in real_vars: + continue + assert doc_param in real_parms, f"{doc_param} no found in real params: {real_parms}" + else: + if len(real_parms) > 0: + warnings.warn(f"doc is empty, real params: {real_parms}", UserWarning) + + +def create_test_function(cls, func): + if func.__name__ == "__init__": + dynamic_func_name = f"test_{cls.__name__}" + else: + dynamic_func_name = f"test_{cls.__name__}_{func.__name__}" + while dynamic_func_name in global_func_names: + dynamic_func_name = dynamic_func_name + "_" + global_func_names.add(dynamic_func_name) + cls_path = f"{cls.__module__}.{cls.__qualname__}" + func_path = f"{cls_path}.{func.__name__}" + code = f"def {dynamic_func_name}():\n do_check_method({cls_path}, {func_path})" + exec(code, globals()) + + +def gen_check_cls_and_funtions(): + all_classes = get_sub_classes(lazyllm) + target_list = [] + for cls in all_classes: + all_methods = inspect.getmembers(cls, predicate=inspect.isfunction) + custom_methods = [func for name, func in all_methods if not name.startswith('_') or name == '__init__'] + overridden_methods = [func for func in custom_methods if is_method_overridden(cls, func)] + for overridden_method in overridden_methods: + create_test_function(cls, overridden_method) + + +global_func_names = set() +gen_check_cls_and_funtions() diff --git a/tests/doc_check/doc_example_check.py b/tests/doc_check/doc_example_check.py new file mode 100644 index 00000000..4f07a2b0 --- /dev/null +++ b/tests/doc_check/doc_example_check.py @@ -0,0 +1,65 @@ +import lazyllm +from pathlib import Path +import pytest +from typing import Union +import re + + +global_func_names = set() +pattern = re.compile(r'^(add_english_doc\(|add_chinese_doc\(|add_example\()') + + +def add_chinese_doc(obj_name, docstr, module=lazyllm): + pass + + +def add_english_doc(obj_name, docstr, module=lazyllm): + pass + + +def add_example(obj_name, docstr: Union[str, list], module=lazyllm): + func_name = "test_" + obj_name.replace(".", "_") + while func_name in global_func_names: + func_name = func_name + "_" + global_func_names.add(func_name) + + if isinstance(docstr, list): + lines = [d for doc in docstr for d in doc.split("\n")] + elif isinstance(docstr, str): + lines = docstr.split("\n") + else: + raise TypeError("Expected str or list, got %s" % type(docstr)) + code_lines = [] + for line in lines: + if line.startswith(">>> ") or line.startswith("... "): + code_lines.append(f" {line[4:]}") + if len(code_lines) == 0: + return + func_code = f"def {func_name}():\n" + "\n".join(code_lines) + lazyllm.LOG.info(f"\nTest example:\n{func_code}") + exec(func_code, globals()) + + +def process_doc(doc_file): + with open(doc_file, "r", encoding="utf-8") as f: + doc_lines = f.readlines() + st_idx = 0 + for i in range(len(doc_lines)): + match = pattern.match(doc_lines[i]) + if match: + st_idx = i + break + if st_idx == len(doc_lines): + return + doc_part = ''.join(doc_lines[st_idx:]) + exec(doc_part, globals()) + + +# 先用一个运行快的例子试一下 +doc_files = Path("lazyllm/docs/").glob("flow.py") +for doc_file in doc_files: + process_doc(doc_file) + + +if __name__ == "__main__": + pytest.main()