From 71a3cdf80c0d12aadca6f9b0f5a43883649bdbea Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 12 Jan 2022 11:49:53 +0000 Subject: bpf/scripts: Raise an exception if the correct number of helpers are not generated Currently bpf_helper_defs.h and the bpf helpers man page are auto-generated using function documentation present in bpf.h. If the documentation for the helper is missing or doesn't follow a specific format for e.g. if a function is documented as: * long bpf_kallsyms_lookup_name( const char *name, int name_sz, int flags, u64 *res ) instead of * long bpf_kallsyms_lookup_name(const char *name, int name_sz, int flags, u64 *res) (notice the extra space at the start and end of function arguments) then that helper is not dumped in the auto-generated header and results in an invalid call during eBPF runtime, even if all the code specific to the helper is correct. This patch checks the number of functions documented within the header file with those present as part of #define __BPF_FUNC_MAPPER and raises an Exception if they don't match. It is not needed with the currently documented upstream functions, but can help in debugging when developing new helpers when there might be missing or misformatted documentation. Signed-off-by: Usama Arif Signed-off-by: Andrii Nakryiko Reviewed-by: Quentin Monnet Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20220112114953.722380-1-usama.arif@bytedance.com --- scripts/bpf_doc.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index a6403ddf5de7..5cf8ae2e72bd 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -87,6 +87,8 @@ class HeaderParser(object): self.line = '' self.helpers = [] self.commands = [] + self.desc_unique_helpers = set() + self.define_unique_helpers = [] def parse_element(self): proto = self.parse_symbol() @@ -193,19 +195,42 @@ class HeaderParser(object): except NoSyscallCommandFound: break - def parse_helpers(self): + def parse_desc_helpers(self): self.seek_to('* Start of BPF helper function descriptions:', 'Could not find start of eBPF helper descriptions list') while True: try: helper = self.parse_helper() self.helpers.append(helper) + proto = helper.proto_break_down() + self.desc_unique_helpers.add(proto['name']) except NoHelperFound: break + def parse_define_helpers(self): + # Parse the number of FN(...) in #define __BPF_FUNC_MAPPER to compare + # later with the number of unique function names present in description. + # Note: seek_to(..) discards the first line below the target search text, + # resulting in FN(unspec) being skipped and not added to self.define_unique_helpers. + self.seek_to('#define __BPF_FUNC_MAPPER(FN)', + 'Could not find start of eBPF helper definition list') + # Searches for either one or more FN(\w+) defines or a backslash for newline + p = re.compile('\s*(FN\(\w+\))+|\\\\') + fn_defines_str = '' + while True: + capture = p.match(self.line) + if capture: + fn_defines_str += self.line + else: + break + self.line = self.reader.readline() + # Find the number of occurences of FN(\w+) + self.define_unique_helpers = re.findall('FN\(\w+\)', fn_defines_str) + def run(self): self.parse_syscall() - self.parse_helpers() + self.parse_desc_helpers() + self.parse_define_helpers() self.reader.close() ############################################################################### @@ -295,6 +320,25 @@ class PrinterRST(Printer): print('') +def helper_number_check(desc_unique_helpers, define_unique_helpers): + """ + Checks the number of functions documented within the header file + with those present as part of #define __BPF_FUNC_MAPPER and raise an + Exception if they don't match. + """ + nr_desc_unique_helpers = len(desc_unique_helpers) + nr_define_unique_helpers = len(define_unique_helpers) + if nr_desc_unique_helpers != nr_define_unique_helpers: + helper_exception = ''' +The number of unique helpers in description (%d) doesn\'t match the number of unique helpers defined in __BPF_FUNC_MAPPER (%d) +''' % (nr_desc_unique_helpers, nr_define_unique_helpers) + if nr_desc_unique_helpers < nr_define_unique_helpers: + # Function description is parsed until no helper is found (which can be due to + # misformatting). Hence, only print the first missing/misformatted function. + helper_exception += ''' +The description for %s is not present or formatted correctly. +''' % (define_unique_helpers[nr_desc_unique_helpers]) + raise Exception(helper_exception) class PrinterHelpersRST(PrinterRST): """ @@ -305,6 +349,7 @@ class PrinterHelpersRST(PrinterRST): """ def __init__(self, parser): self.elements = parser.helpers + helper_number_check(parser.desc_unique_helpers, parser.define_unique_helpers) def print_header(self): header = '''\ @@ -509,6 +554,7 @@ class PrinterHelpers(Printer): """ def __init__(self, parser): self.elements = parser.helpers + helper_number_check(parser.desc_unique_helpers, parser.define_unique_helpers) type_fwds = [ 'struct bpf_fib_lookup', -- cgit v1.2.3 From f1f3f67fd8ed6f512955bbbc76b04e9dc33ddeb6 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 19 Jan 2022 11:44:41 +0000 Subject: bpf/scripts: Make description and returns section for helpers/syscalls mandatory This enforce a minimal formatting consistency for the documentation. The description and returns missing for a few helpers have also been added. Signed-off-by: Usama Arif Signed-off-by: Andrii Nakryiko Reviewed-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20220119114442.1452088-2-usama.arif@bytedance.com --- scripts/bpf_doc.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) (limited to 'scripts') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index 5cf8ae2e72bd..20441e5d2d33 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -92,14 +92,14 @@ class HeaderParser(object): def parse_element(self): proto = self.parse_symbol() - desc = self.parse_desc() - ret = self.parse_ret() + desc = self.parse_desc(proto) + ret = self.parse_ret(proto) return APIElement(proto=proto, desc=desc, ret=ret) def parse_helper(self): proto = self.parse_proto() - desc = self.parse_desc() - ret = self.parse_ret() + desc = self.parse_desc(proto) + ret = self.parse_ret(proto) return Helper(proto=proto, desc=desc, ret=ret) def parse_symbol(self): @@ -129,16 +129,15 @@ class HeaderParser(object): self.line = self.reader.readline() return capture.group(1) - def parse_desc(self): + def parse_desc(self, proto): p = re.compile(' \* ?(?:\t| {5,8})Description$') capture = p.match(self.line) if not capture: - # Helper can have empty description and we might be parsing another - # attribute: return but do not consume. - return '' + raise Exception("No description section found for " + proto) # Description can be several lines, some of them possibly empty, and it # stops when another subsection title is met. desc = '' + desc_present = False while True: self.line = self.reader.readline() if self.line == ' *\n': @@ -147,21 +146,24 @@ class HeaderParser(object): p = re.compile(' \* ?(?:\t| {5,8})(?:\t| {8})(.*)') capture = p.match(self.line) if capture: + desc_present = True desc += capture.group(1) + '\n' else: break + + if not desc_present: + raise Exception("No description found for " + proto) return desc - def parse_ret(self): + def parse_ret(self, proto): p = re.compile(' \* ?(?:\t| {5,8})Return$') capture = p.match(self.line) if not capture: - # Helper can have empty retval and we might be parsing another - # attribute: return but do not consume. - return '' + raise Exception("No return section found for " + proto) # Return value description can be several lines, some of them possibly # empty, and it stops when another subsection title is met. ret = '' + ret_present = False while True: self.line = self.reader.readline() if self.line == ' *\n': @@ -170,9 +172,13 @@ class HeaderParser(object): p = re.compile(' \* ?(?:\t| {5,8})(?:\t| {8})(.*)') capture = p.match(self.line) if capture: + ret_present = True ret += capture.group(1) + '\n' else: break + + if not ret_present: + raise Exception("No return found for " + proto) return ret def seek_to(self, target, help_message): -- cgit v1.2.3 From 0ba3929e5b3d3fda05d3b9c8d0d20a90a084c19e Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 19 Jan 2022 11:44:42 +0000 Subject: bpf/scripts: Raise an exception if the correct number of sycalls are not generated Currently the syscalls rst and subsequently man page are auto-generated using function documentation present in bpf.h. If the documentation for the syscall is missing or doesn't follow a specific format, then that syscall is not dumped in the auto-generated rst. This patch checks the number of syscalls documented within the header file with those present as part of the enum bpf_cmd and raises an Exception if they don't match. It is not needed with the currently documented upstream syscalls, but can help in debugging when developing new syscalls when there might be missing or misformatted documentation. The function helper_number_check is moved to the Printer parent class and renamed to elem_number_check as all the most derived children classes are using this function now. Signed-off-by: Usama Arif Signed-off-by: Andrii Nakryiko Reviewed-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20220119114442.1452088-3-usama.arif@bytedance.com --- scripts/bpf_doc.py | 86 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 27 deletions(-) (limited to 'scripts') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index 20441e5d2d33..096625242475 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -89,6 +89,8 @@ class HeaderParser(object): self.commands = [] self.desc_unique_helpers = set() self.define_unique_helpers = [] + self.desc_syscalls = [] + self.enum_syscalls = [] def parse_element(self): proto = self.parse_symbol() @@ -103,7 +105,7 @@ class HeaderParser(object): return Helper(proto=proto, desc=desc, ret=ret) def parse_symbol(self): - p = re.compile(' \* ?(.+)$') + p = re.compile(' \* ?(BPF\w+)$') capture = p.match(self.line) if not capture: raise NoSyscallCommandFound @@ -181,26 +183,55 @@ class HeaderParser(object): raise Exception("No return found for " + proto) return ret - def seek_to(self, target, help_message): + def seek_to(self, target, help_message, discard_lines = 1): self.reader.seek(0) offset = self.reader.read().find(target) if offset == -1: raise Exception(help_message) self.reader.seek(offset) self.reader.readline() - self.reader.readline() + for _ in range(discard_lines): + self.reader.readline() self.line = self.reader.readline() - def parse_syscall(self): + def parse_desc_syscall(self): self.seek_to('* DOC: eBPF Syscall Commands', 'Could not find start of eBPF syscall descriptions list') while True: try: command = self.parse_element() self.commands.append(command) + self.desc_syscalls.append(command.proto) + except NoSyscallCommandFound: break + def parse_enum_syscall(self): + self.seek_to('enum bpf_cmd {', + 'Could not find start of bpf_cmd enum', 0) + # Searches for either one or more BPF\w+ enums + bpf_p = re.compile('\s*(BPF\w+)+') + # Searches for an enum entry assigned to another entry, + # for e.g. BPF_PROG_RUN = BPF_PROG_TEST_RUN, which is + # not documented hence should be skipped in check to + # determine if the right number of syscalls are documented + assign_p = re.compile('\s*(BPF\w+)\s*=\s*(BPF\w+)') + bpf_cmd_str = '' + while True: + capture = assign_p.match(self.line) + if capture: + # Skip line if an enum entry is assigned to another entry + self.line = self.reader.readline() + continue + capture = bpf_p.match(self.line) + if capture: + bpf_cmd_str += self.line + else: + break + self.line = self.reader.readline() + # Find the number of occurences of BPF\w+ + self.enum_syscalls = re.findall('(BPF\w+)+', bpf_cmd_str) + def parse_desc_helpers(self): self.seek_to('* Start of BPF helper function descriptions:', 'Could not find start of eBPF helper descriptions list') @@ -234,7 +265,8 @@ class HeaderParser(object): self.define_unique_helpers = re.findall('FN\(\w+\)', fn_defines_str) def run(self): - self.parse_syscall() + self.parse_desc_syscall() + self.parse_enum_syscall() self.parse_desc_helpers() self.parse_define_helpers() self.reader.close() @@ -266,6 +298,25 @@ class Printer(object): self.print_one(elem) self.print_footer() + def elem_number_check(self, desc_unique_elem, define_unique_elem, type, instance): + """ + Checks the number of helpers/syscalls documented within the header file + description with those defined as part of enum/macro and raise an + Exception if they don't match. + """ + nr_desc_unique_elem = len(desc_unique_elem) + nr_define_unique_elem = len(define_unique_elem) + if nr_desc_unique_elem != nr_define_unique_elem: + exception_msg = ''' +The number of unique %s in description (%d) doesn\'t match the number of unique %s defined in %s (%d) +''' % (type, nr_desc_unique_elem, type, instance, nr_define_unique_elem) + if nr_desc_unique_elem < nr_define_unique_elem: + # Function description is parsed until no helper is found (which can be due to + # misformatting). Hence, only print the first missing/misformatted helper/enum. + exception_msg += ''' +The description for %s is not present or formatted correctly. +''' % (define_unique_elem[nr_desc_unique_elem]) + raise Exception(exception_msg) class PrinterRST(Printer): """ @@ -326,26 +377,6 @@ class PrinterRST(Printer): print('') -def helper_number_check(desc_unique_helpers, define_unique_helpers): - """ - Checks the number of functions documented within the header file - with those present as part of #define __BPF_FUNC_MAPPER and raise an - Exception if they don't match. - """ - nr_desc_unique_helpers = len(desc_unique_helpers) - nr_define_unique_helpers = len(define_unique_helpers) - if nr_desc_unique_helpers != nr_define_unique_helpers: - helper_exception = ''' -The number of unique helpers in description (%d) doesn\'t match the number of unique helpers defined in __BPF_FUNC_MAPPER (%d) -''' % (nr_desc_unique_helpers, nr_define_unique_helpers) - if nr_desc_unique_helpers < nr_define_unique_helpers: - # Function description is parsed until no helper is found (which can be due to - # misformatting). Hence, only print the first missing/misformatted function. - helper_exception += ''' -The description for %s is not present or formatted correctly. -''' % (define_unique_helpers[nr_desc_unique_helpers]) - raise Exception(helper_exception) - class PrinterHelpersRST(PrinterRST): """ A printer for dumping collected information about helpers as a ReStructured @@ -355,7 +386,7 @@ class PrinterHelpersRST(PrinterRST): """ def __init__(self, parser): self.elements = parser.helpers - helper_number_check(parser.desc_unique_helpers, parser.define_unique_helpers) + self.elem_number_check(parser.desc_unique_helpers, parser.define_unique_helpers, 'helper', '__BPF_FUNC_MAPPER') def print_header(self): header = '''\ @@ -529,6 +560,7 @@ class PrinterSyscallRST(PrinterRST): """ def __init__(self, parser): self.elements = parser.commands + self.elem_number_check(parser.desc_syscalls, parser.enum_syscalls, 'syscall', 'bpf_cmd') def print_header(self): header = '''\ @@ -560,7 +592,7 @@ class PrinterHelpers(Printer): """ def __init__(self, parser): self.elements = parser.helpers - helper_number_check(parser.desc_unique_helpers, parser.define_unique_helpers) + self.elem_number_check(parser.desc_unique_helpers, parser.define_unique_helpers, 'helper', '__BPF_FUNC_MAPPER') type_fwds = [ 'struct bpf_fib_lookup', -- cgit v1.2.3 From 613fe169237785a4bb1d06397b52606b2967da53 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 1 Feb 2022 13:56:21 -0700 Subject: kbuild: Add CONFIG_PAHOLE_VERSION There are a few different places where pahole's version is turned into a three digit form with the exact same command. Move this command into scripts/pahole-version.sh to reduce the amount of duplication across the tree. Create CONFIG_PAHOLE_VERSION so the version code can be used in Kconfig to enable and disable configuration options based on the pahole version, which is already done in a couple of places. Signed-off-by: Nathan Chancellor Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220201205624.652313-3-nathan@kernel.org --- scripts/pahole-version.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 scripts/pahole-version.sh (limited to 'scripts') diff --git a/scripts/pahole-version.sh b/scripts/pahole-version.sh new file mode 100755 index 000000000000..f8a32ab93ad1 --- /dev/null +++ b/scripts/pahole-version.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Usage: $ ./pahole-version.sh pahole +# +# Prints pahole's version in a 3-digit form, such as 119 for v1.19. + +if [ ! -x "$(command -v "$@")" ]; then + echo 0 + exit 1 +fi + +"$@" --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/' -- cgit v1.2.3 From 2d6c9810eb8915c4ddede707b8e167a1d919e1ca Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 1 Feb 2022 13:56:22 -0700 Subject: scripts/pahole-flags.sh: Use pahole-version.sh Use pahole-version.sh to get pahole's version code to reduce the amount of duplication across the tree. Signed-off-by: Nathan Chancellor Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220201205624.652313-4-nathan@kernel.org --- scripts/pahole-flags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/pahole-flags.sh b/scripts/pahole-flags.sh index e6093adf4c06..c293941612e7 100755 --- a/scripts/pahole-flags.sh +++ b/scripts/pahole-flags.sh @@ -7,7 +7,7 @@ if ! [ -x "$(command -v ${PAHOLE})" ]; then exit 0 fi -pahole_ver=$(${PAHOLE} --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/') +pahole_ver=$($(dirname $0)/pahole-version.sh ${PAHOLE}) if [ "${pahole_ver}" -ge "118" ] && [ "${pahole_ver}" -le "121" ]; then # pahole 1.18 through 1.21 can't handle zero-sized per-CPU vars -- cgit v1.2.3 From b4f72786429cc57383ff41e02407726313ca178d Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Thu, 17 Feb 2022 09:54:27 -0800 Subject: scripts/pahole-flags.sh: Parse DWARF and generate BTF with multithreading. Pass a "-j" argument to pahole if possible to reduce the time of generating BTF info. Since v1.22, pahole can parse DWARF and generate BTF with multithreading to speed up the conversion. It will reduce the overall build time of the kernel for seconds. v3 fixes whitespaces and improves the commit description. v2 checks the version of pahole to enable multithreading only if possible. [v2] https://lore.kernel.org/bpf/20220216193431.2691015-1-kuifeng@fb.com/ [v1] https://lore.kernel.org/bpf/20220216004616.2079689-1-kuifeng@fb.com/ Signed-off-by: Kui-Feng Lee Signed-off-by: Andrii Nakryiko Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20220217175427.649713-1-kuifeng@fb.com --- scripts/pahole-flags.sh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/pahole-flags.sh b/scripts/pahole-flags.sh index c293941612e7..0d99ef17e4a5 100755 --- a/scripts/pahole-flags.sh +++ b/scripts/pahole-flags.sh @@ -16,5 +16,8 @@ fi if [ "${pahole_ver}" -ge "121" ]; then extra_paholeopt="${extra_paholeopt} --btf_gen_floats" fi +if [ "${pahole_ver}" -ge "122" ]; then + extra_paholeopt="${extra_paholeopt} -j" +fi echo ${extra_paholeopt} -- cgit v1.2.3