From 29efbb24d992564db4bbb808597719934ed9ac9f Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 24 Sep 2019 16:29:58 -0700 Subject: docs: Use make invocation's -j argument for parallelism While sphinx 1.7 and later supports "-jauto" for parallelism, this effectively ignores the "-j" flag used in the "make" invocation, which may cause confusion for build systems. Instead, extract the available parallelism from "make"'s job server (since it is not exposed in any special variables) and use that for the "sphinx-build" run. Now things work correctly for builds where -j is specified at the top-level: make -j16 htmldocs If -j is not specified, continue to fallback to "-jauto" if available. Signed-off-by: Kees Cook Signed-off-by: Jonathan Corbet --- scripts/jobserver-count | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 scripts/jobserver-count (limited to 'scripts') diff --git a/scripts/jobserver-count b/scripts/jobserver-count new file mode 100755 index 000000000000..0b482d6884d2 --- /dev/null +++ b/scripts/jobserver-count @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0+ +# +# This determines how many parallel tasks "make" is expecting, as it is +# not exposed via an special variables. +# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver +from __future__ import print_function +import os, sys, fcntl, errno + +# Default parallelism is "1" unless overridden on the command-line. +default="1" +if len(sys.argv) > 1: + default=sys.argv[1] + +# Set non-blocking for a given file descriptor. +def nonblock(fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + return fd + +# Extract and prepare jobserver file descriptors from envirnoment. +try: + # Fetch the make environment options. + flags = os.environ['MAKEFLAGS'] + + # Look for "--jobserver=R,W" + opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] + + # Parse out R,W file descriptor numbers and set them nonblocking. + fds = opts[0].split("=", 1)[1] + reader, writer = [int(x) for x in fds.split(",", 1)] + reader = nonblock(reader) +except (KeyError, IndexError, ValueError, IOError): + # Any missing environment strings or bad fds should result in just + # using the default specified parallelism. + print(default) + sys.exit(0) + +# Read out as many jobserver slots as possible. +jobs = b"" +while True: + try: + slot = os.read(reader, 1) + jobs += slot + except (OSError, IOError) as e: + if e.errno == errno.EWOULDBLOCK: + # Stop when reach the end of the jobserver queue. + break + raise e +# Return all the reserved slots. +os.write(writer, jobs) + +# If the jobserver was (impossibly) full or communication failed, use default. +if len(jobs) < 1: + print(default) + +# Report available slots (with a bump for our caller's reserveration). +print(len(jobs) + 1) -- cgit v1.2.3 From 2730ce017fa6df49bad9ad932932b863f63a4b50 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Wed, 18 Sep 2019 18:37:54 -0600 Subject: scripts/sphinx-pre-install: add how to exit virtualenv usage message Add usage message on how to exit the virtualenv after documentation work is done. Signed-off-by: Shuah Khan Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 3b638c0e1a4f..013099cd120d 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -645,6 +645,12 @@ sub check_distros() # Common dependencies # +sub deactivate_help() +{ + printf "\tIf you want to exit the virtualenv, you can use:\n"; + printf "\tdeactivate\n"; +} + sub check_needs() { # Check for needed programs/tools @@ -686,6 +692,7 @@ sub check_needs() if ($need_sphinx && scalar @activates > 0 && $activates[0] ge $min_activate) { printf "\nNeed to activate a compatible Sphinx version on virtualenv with:\n"; printf "\t. $activates[0]\n"; + deactivate_help(); exit (1); } else { my $rec_activate = "$virtenv_dir/bin/activate"; @@ -697,6 +704,7 @@ sub check_needs() printf "\t$virtualenv $virtenv_dir\n"; printf "\t. $rec_activate\n"; printf "\tpip install -r $requirement_file\n"; + deactivate_help(); $need++ if (!$rec_sphinx_upgrade); } -- cgit v1.2.3 From 2b5f78e5e942d76e5497f53c2298900224b52c51 Mon Sep 17 00:00:00 2001 From: André Almeida Date: Tue, 17 Sep 2019 16:41:45 -0300 Subject: kernel-doc: fix processing nested structs with attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current regular expression for strip attributes of structs (and for nested ones as well) also removes all whitespaces that may surround the attribute. After that, the code will split structs and iterate for each symbol separated by comma at the end of struct definition (e.g. "} alias1, alias2;"). However, if the nested struct does not have any alias and has an attribute, it will result in a empty string at the closing bracket (e.g "};"). This will make the split return nothing and $newmember will keep uninitialized. Fix that, by ensuring that the attribute substitution will leave at least one whitespace. Signed-off-by: André Almeida Signed-off-by: Jonathan Corbet --- scripts/kernel-doc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 81dc91760b23..baa2be7e5284 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1073,10 +1073,10 @@ sub dump_struct($$) { # strip comments: $members =~ s/\/\*.*?\*\///gos; # strip attributes - $members =~ s/\s*__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)//gi; - $members =~ s/\s*__aligned\s*\([^;]*\)//gos; - $members =~ s/\s*__packed\s*//gos; - $members =~ s/\s*CRYPTO_MINALIGN_ATTR//gos; + $members =~ s/\s*__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)/ /gi; + $members =~ s/\s*__aligned\s*\([^;]*\)/ /gos; + $members =~ s/\s*__packed\s*/ /gos; + $members =~ s/\s*CRYPTO_MINALIGN_ATTR/ /gos; # replace DECLARE_BITMAP $members =~ s/DECLARE_BITMAP\s*\(([^,)]+),\s*([^,)]+)\)/unsigned long $1\[BITS_TO_LONGS($2)\]/gos; # replace DECLARE_HASHTABLE -- cgit v1.2.3 From f861537d5f856f8bffc7ddd1f9c1a59bfed0012a Mon Sep 17 00:00:00 2001 From: André Almeida Date: Tue, 17 Sep 2019 16:41:46 -0300 Subject: kernel-doc: add support for ____cacheline_aligned_in_smp attribute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Subroutine dump_struct uses type attributes to check if the struct syntax is valid. Then, it removes all attributes before using it for output. `____cacheline_aligned_in_smp` is an attribute that is not included in both steps. Add it, since it is used by kernel structs. Signed-off-by: André Almeida Signed-off-by: Jonathan Corbet --- scripts/kernel-doc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/kernel-doc b/scripts/kernel-doc index baa2be7e5284..a529375c8536 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1062,7 +1062,7 @@ sub dump_struct($$) { my $x = shift; my $file = shift; - if ($x =~ /(struct|union)\s+(\w+)\s*\{(.*)\}(\s*(__packed|__aligned|__attribute__\s*\(\([a-z0-9,_\s\(\)]*\)\)))*/) { + if ($x =~ /(struct|union)\s+(\w+)\s*\{(.*)\}(\s*(__packed|__aligned|____cacheline_aligned_in_smp|__attribute__\s*\(\([a-z0-9,_\s\(\)]*\)\)))*/) { my $decl_type = $1; $declaration_name = $2; my $members = $3; @@ -1077,6 +1077,7 @@ sub dump_struct($$) { $members =~ s/\s*__aligned\s*\([^;]*\)/ /gos; $members =~ s/\s*__packed\s*/ /gos; $members =~ s/\s*CRYPTO_MINALIGN_ATTR/ /gos; + $members =~ s/\s*____cacheline_aligned_in_smp/ /gos; # replace DECLARE_BITMAP $members =~ s/DECLARE_BITMAP\s*\(([^,)]+),\s*([^,)]+)\)/unsigned long $1\[BITS_TO_LONGS($2)\]/gos; # replace DECLARE_HASHTABLE -- cgit v1.2.3 From ff8fdb36ac35ee90388fb3f5fdac679b25765841 Mon Sep 17 00:00:00 2001 From: Jeremy MAURO Date: Wed, 2 Oct 2019 15:33:39 +0200 Subject: scripts/sphinx-pre-install: allow checking for multiple missing files The current implementation take a simple file as first argument, this change allows to take a list as a first argument. Some file could have a different path according distribution version Signed-off-by: Jeremy MAURO Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 013099cd120d..646b97790e1a 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -124,11 +124,13 @@ sub add_package($$) sub check_missing_file($$$) { - my $file = shift; + my $files = shift; my $package = shift; my $is_optional = shift; - return if(-e $file); + for (@$files) { + return if(-e $_); + } add_package($package, $is_optional); } @@ -343,10 +345,10 @@ sub give_debian_hints() ); if ($pdf) { - check_missing_file("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"], "fonts-dejavu", 2); - check_missing_file("/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], "fonts-noto-cjk", 2); } @@ -413,7 +415,7 @@ sub give_redhat_hints() } if ($pdf) { - check_missing_file("/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc"], "google-noto-sans-cjk-ttc-fonts", 2); } @@ -498,7 +500,7 @@ sub give_mageia_hints() $map{"latexmk"} = "texlive-collection-basic"; if ($pdf) { - check_missing_file("/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc"], "google-noto-sans-cjk-ttc-fonts", 2); } @@ -528,7 +530,7 @@ sub give_arch_linux_hints() check_pacman_missing(\@archlinux_tex_pkgs, 2) if ($pdf); if ($pdf) { - check_missing_file("/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], "noto-fonts-cjk", 2); } @@ -549,11 +551,11 @@ sub give_gentoo_hints() "rsvg-convert" => "gnome-base/librsvg", ); - check_missing_file("/usr/share/fonts/dejavu/DejaVuSans.ttf", + check_missing_file(["/usr/share/fonts/dejavu/DejaVuSans.ttf"], "media-fonts/dejavu", 2) if ($pdf); if ($pdf) { - check_missing_file("/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf"], "media-fonts/noto-cjk", 2); } -- cgit v1.2.3 From 9692f2fdb163a4a5e79ddb9b7e0f15d30384c7c2 Mon Sep 17 00:00:00 2001 From: Jeremy MAURO Date: Wed, 2 Oct 2019 15:35:42 +0200 Subject: scripts/sphinx-pre-install: Add a new path for the debian package "fonts-noto-cjk" The latest debian version "bullseye/sid" has changed the path of the file "notoserifcjk-regular.ttc", with the previous change and this change we keep the backward compatibility and add the latest debian version Signed-off-by: Jeremy MAURO Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 646b97790e1a..68385fa62ff4 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -348,7 +348,8 @@ sub give_debian_hints() check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"], "fonts-dejavu", 2); - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc"], "fonts-noto-cjk", 2); } -- cgit v1.2.3 From 43756e347f213b68f884c3b4082e95e7f98204f1 Mon Sep 17 00:00:00 2001 From: Jonathan Neuschäfer Date: Thu, 7 Nov 2019 14:41:33 +0100 Subject: scripts/kernel-doc: Add support for named variable macro arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, when kernel-doc encounters a macro with a named variable argument[1], such as this: #define hlist_for_each_entry_rcu(pos, head, member, cond...) ... it expects the variable argument to be documented as `cond...`, rather than `cond`. This is semantically wrong, because the name (as used in the macro body) is actually `cond`. With this patch, kernel-doc will accept the name without dots (`cond` in the example above) in doc comments, and warn if the name with dots (`cond...`) is used and verbose mode[2] is enabled. The support for the `cond...` syntax can be removed later, when the documentation of all such macros has been switched to the new syntax. Testing this patch on top of v5.4-rc6, `make htmldocs` shows a few changes in log output and HTML output: 1) The following warnings[3] are eliminated: ./include/linux/rculist.h:374: warning: Excess function parameter 'cond' description in 'list_for_each_entry_rcu' ./include/linux/rculist.h:651: warning: Excess function parameter 'cond' description in 'hlist_for_each_entry_rcu' 2) For list_for_each_entry_rcu and hlist_for_each_entry_rcu, the correct description is shown 3) Named variable arguments are shown without dots [1]: https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html [2]: scripts/kernel-doc -v [3]: See also https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git/commit/?h=dev&id=5bc4bc0d6153617eabde275285b7b5a8137fdf3c Signed-off-by: Jonathan Neuschäfer Tested-by: Paul E. McKenney Signed-off-by: Jonathan Corbet --- scripts/kernel-doc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'scripts') diff --git a/scripts/kernel-doc b/scripts/kernel-doc index a529375c8536..f2d73f04e71d 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1450,6 +1450,10 @@ sub push_parameter($$$$) { # handles unnamed variable parameters $param = "..."; } + elsif ($param =~ /\w\.\.\.$/) { + # for named variable parameters of the form `x...`, remove the dots + $param =~ s/\.\.\.$//; + } if (!defined $parameterdescs{$param} || $parameterdescs{$param} eq "") { $parameterdescs{$param} = "variable arguments"; } @@ -1937,6 +1941,18 @@ sub process_name($$) { sub process_body($$) { my $file = shift; + # Until all named variable macro parameters are + # documented using the bare name (`x`) rather than with + # dots (`x...`), strip the dots: + if ($section =~ /\w\.\.\.$/) { + $section =~ s/\.\.\.$//; + + if ($verbose) { + print STDERR "${file}:$.: warning: Variable macro arguments should be documented without dots\n"; + ++$warnings; + } + } + if (/$doc_sect/i) { # case insensitive for supported section names $newsection = $1; $newcontents = $2; -- cgit v1.2.3 From 0d0da9aa03a178d343f64f3bd7d545b0d3bf8b7e Mon Sep 17 00:00:00 2001 From: Louis Taylor Date: Sat, 2 Nov 2019 18:45:11 +0000 Subject: scripts/sphinx-pre-install: fix Arch latexmk dependency On Arch Linux, latexmk is installed in the texlive-core package. Signed-off-by: Louis Taylor Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 68385fa62ff4..470ccfe678aa 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -520,6 +520,7 @@ sub give_arch_linux_hints() "dot" => "graphviz", "convert" => "imagemagick", "xelatex" => "texlive-bin", + "latexmk" => "texlive-core", "rsvg-convert" => "extra/librsvg", ); -- cgit v1.2.3 From 4920323cffc0fe10ac107ab3e94d1bd218a678d9 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 21 Nov 2019 12:59:27 -0800 Subject: docs, parallelism: Fix failure path and add comment Rasmus noted that the failure path didn't correctly exit. Fix this and add another comment about GNU Make's job server environment variable names over time. Reported-by: Rasmus Villemoes Link: https://lore.kernel.org/lkml/eb25959a-9ec4-3530-2031-d9d716b40b20@rasmusvillemoes.dk Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20191121205929.40371-2-keescook@chromium.org Signed-off-by: Jonathan Corbet --- scripts/jobserver-count | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/jobserver-count b/scripts/jobserver-count index 0b482d6884d2..6e15b38df3d0 100755 --- a/scripts/jobserver-count +++ b/scripts/jobserver-count @@ -24,6 +24,8 @@ try: flags = os.environ['MAKEFLAGS'] # Look for "--jobserver=R,W" + # Note that GNU Make has used --jobserver-fds and --jobserver-auth + # so this handles all of them. opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] # Parse out R,W file descriptor numbers and set them nonblocking. @@ -53,6 +55,7 @@ os.write(writer, jobs) # If the jobserver was (impossibly) full or communication failed, use default. if len(jobs) < 1: print(default) + sys.exit(0) # Report available slots (with a bump for our caller's reserveration). print(len(jobs) + 1) -- cgit v1.2.3 From dffd011480d727a819c537edf3b90ef063731725 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 21 Nov 2019 12:59:28 -0800 Subject: docs, parallelism: Do not leak blocking mode to other readers Setting non-blocking via a local copy of the jobserver file descriptor is safer than just assuming other reader processes with the same fd open are prepared for it to be non-blocking. Suggested-by: Rasmus Villemoes Link: https://lore.kernel.org/lkml/44c01043-ab24-b4de-6544-e8efd153e27a@rasmusvillemoes.dk Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20191121205929.40371-3-keescook@chromium.org Signed-off-by: Jonathan Corbet --- scripts/jobserver-count | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/jobserver-count b/scripts/jobserver-count index 6e15b38df3d0..7807bfa7dafa 100755 --- a/scripts/jobserver-count +++ b/scripts/jobserver-count @@ -12,12 +12,6 @@ default="1" if len(sys.argv) > 1: default=sys.argv[1] -# Set non-blocking for a given file descriptor. -def nonblock(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - return fd - # Extract and prepare jobserver file descriptors from envirnoment. try: # Fetch the make environment options. @@ -31,8 +25,12 @@ try: # Parse out R,W file descriptor numbers and set them nonblocking. fds = opts[0].split("=", 1)[1] reader, writer = [int(x) for x in fds.split(",", 1)] - reader = nonblock(reader) -except (KeyError, IndexError, ValueError, IOError): + # Open a private copy of reader to avoid setting nonblocking + # on an unexpecting process with the same reader fd. + reader = os.open("/proc/self/fd/%d" % (reader), + os.O_RDONLY | os.O_NONBLOCK) +except (KeyError, IndexError, ValueError, IOError, OSError) as e: + print(e, file=sys.stderr) # Any missing environment strings or bad fds should result in just # using the default specified parallelism. print(default) -- cgit v1.2.3 From 51e46c7a4007d271b2d42dbc2df953ab968577a7 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 21 Nov 2019 12:59:29 -0800 Subject: docs, parallelism: Rearrange how jobserver reservations are made Rasmus correctly observed that the existing jobserver reservation only worked if no other build targets were specified. The correct approach is to hold the jobserver slots until sphinx has finished. To fix this, the following changes are made: - refactor (and rename) scripts/jobserver-exec to set an environment variable for the maximally reserved jobserver slots and exec a child, to release the slots on exit. - create Documentation/scripts/parallel-wrapper.sh which examines both $PARALLELISM and the detected "-jauto" logic from Documentation/Makefile to decide sphinx's final -j argument. - chain these together in Documentation/Makefile Suggested-by: Rasmus Villemoes Link: https://lore.kernel.org/lkml/eb25959a-9ec4-3530-2031-d9d716b40b20@rasmusvillemoes.dk Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20191121205929.40371-4-keescook@chromium.org Signed-off-by: Jonathan Corbet --- Documentation/Makefile | 5 +-- Documentation/sphinx/parallel-wrapper.sh | 33 ++++++++++++++++ scripts/jobserver-count | 59 ---------------------------- scripts/jobserver-exec | 66 ++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 62 deletions(-) create mode 100644 Documentation/sphinx/parallel-wrapper.sh delete mode 100755 scripts/jobserver-count create mode 100755 scripts/jobserver-exec (limited to 'scripts') diff --git a/Documentation/Makefile b/Documentation/Makefile index ce8eb63b523a..30554a2fbdd7 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -33,8 +33,6 @@ ifeq ($(HAVE_SPHINX),0) else # HAVE_SPHINX -export SPHINX_PARALLEL = $(shell perl -e 'open IN,"sphinx-build --version 2>&1 |"; while () { if (m/([\d\.]+)/) { print "auto" if ($$1 >= "1.7") } ;} close IN') - # User-friendly check for pdflatex and latexmk HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi) HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi) @@ -67,8 +65,9 @@ quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4) cmd_sphinx = $(MAKE) BUILDDIR=$(abspath $(BUILDDIR)) $(build)=Documentation/media $2 && \ PYTHONDONTWRITEBYTECODE=1 \ BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \ + $(PYTHON) $(srctree)/scripts/jobserver-exec \ + $(SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \ $(SPHINXBUILD) \ - -j $(shell python $(srctree)/scripts/jobserver-count $(SPHINX_PARALLEL)) \ -b $2 \ -c $(abspath $(srctree)/$(src)) \ -d $(abspath $(BUILDDIR)/.doctrees/$3) \ diff --git a/Documentation/sphinx/parallel-wrapper.sh b/Documentation/sphinx/parallel-wrapper.sh new file mode 100644 index 000000000000..7daf5133bdd3 --- /dev/null +++ b/Documentation/sphinx/parallel-wrapper.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ +# +# Figure out if we should follow a specific parallelism from the make +# environment (as exported by scripts/jobserver-exec), or fall back to +# the "auto" parallelism when "-jN" is not specified at the top-level +# "make" invocation. + +sphinx="$1" +shift || true + +parallel="$PARALLELISM" +if [ -z "$parallel" ] ; then + # If no parallelism is specified at the top-level make, then + # fall back to the expected "-jauto" mode that the "htmldocs" + # target has had. + auto=$(perl -e 'open IN,"'"$sphinx"' --version 2>&1 |"; + while () { + if (m/([\d\.]+)/) { + print "auto" if ($1 >= "1.7") + } + } + close IN') + if [ -n "$auto" ] ; then + parallel="$auto" + fi +fi +# Only if some parallelism has been determined do we add the -jN option. +if [ -n "$parallel" ] ; then + parallel="-j$parallel" +fi + +exec "$sphinx" "$parallel" "$@" diff --git a/scripts/jobserver-count b/scripts/jobserver-count deleted file mode 100755 index 7807bfa7dafa..000000000000 --- a/scripts/jobserver-count +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: GPL-2.0+ -# -# This determines how many parallel tasks "make" is expecting, as it is -# not exposed via an special variables. -# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver -from __future__ import print_function -import os, sys, fcntl, errno - -# Default parallelism is "1" unless overridden on the command-line. -default="1" -if len(sys.argv) > 1: - default=sys.argv[1] - -# Extract and prepare jobserver file descriptors from envirnoment. -try: - # Fetch the make environment options. - flags = os.environ['MAKEFLAGS'] - - # Look for "--jobserver=R,W" - # Note that GNU Make has used --jobserver-fds and --jobserver-auth - # so this handles all of them. - opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] - - # Parse out R,W file descriptor numbers and set them nonblocking. - fds = opts[0].split("=", 1)[1] - reader, writer = [int(x) for x in fds.split(",", 1)] - # Open a private copy of reader to avoid setting nonblocking - # on an unexpecting process with the same reader fd. - reader = os.open("/proc/self/fd/%d" % (reader), - os.O_RDONLY | os.O_NONBLOCK) -except (KeyError, IndexError, ValueError, IOError, OSError) as e: - print(e, file=sys.stderr) - # Any missing environment strings or bad fds should result in just - # using the default specified parallelism. - print(default) - sys.exit(0) - -# Read out as many jobserver slots as possible. -jobs = b"" -while True: - try: - slot = os.read(reader, 1) - jobs += slot - except (OSError, IOError) as e: - if e.errno == errno.EWOULDBLOCK: - # Stop when reach the end of the jobserver queue. - break - raise e -# Return all the reserved slots. -os.write(writer, jobs) - -# If the jobserver was (impossibly) full or communication failed, use default. -if len(jobs) < 1: - print(default) - sys.exit(0) - -# Report available slots (with a bump for our caller's reserveration). -print(len(jobs) + 1) diff --git a/scripts/jobserver-exec b/scripts/jobserver-exec new file mode 100755 index 000000000000..0fdb31a790a8 --- /dev/null +++ b/scripts/jobserver-exec @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0+ +# +# This determines how many parallel tasks "make" is expecting, as it is +# not exposed via an special variables, reserves them all, runs a subprocess +# with PARALLELISM environment variable set, and releases the jobs back again. +# +# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver +from __future__ import print_function +import os, sys, errno +import subprocess + +# Extract and prepare jobserver file descriptors from envirnoment. +claim = 0 +jobs = b"" +try: + # Fetch the make environment options. + flags = os.environ['MAKEFLAGS'] + + # Look for "--jobserver=R,W" + # Note that GNU Make has used --jobserver-fds and --jobserver-auth + # so this handles all of them. + opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] + + # Parse out R,W file descriptor numbers and set them nonblocking. + fds = opts[0].split("=", 1)[1] + reader, writer = [int(x) for x in fds.split(",", 1)] + # Open a private copy of reader to avoid setting nonblocking + # on an unexpecting process with the same reader fd. + reader = os.open("/proc/self/fd/%d" % (reader), + os.O_RDONLY | os.O_NONBLOCK) + + # Read out as many jobserver slots as possible. + while True: + try: + slot = os.read(reader, 8) + jobs += slot + except (OSError, IOError) as e: + if e.errno == errno.EWOULDBLOCK: + # Stop at the end of the jobserver queue. + break + # If something went wrong, give back the jobs. + if len(jobs): + os.write(writer, jobs) + raise e + # Add a bump for our caller's reserveration, since we're just going + # to sit here blocked on our child. + claim = len(jobs) + 1 +except (KeyError, IndexError, ValueError, OSError, IOError) as e: + # Any missing environment strings or bad fds should result in just + # not being parallel. + pass + +# We can only claim parallelism if there was a jobserver (i.e. a top-level +# "-jN" argument) and there were no other failures. Otherwise leave out the +# environment variable and let the child figure out what is best. +if claim > 0: + os.environ['PARALLELISM'] = '%d' % (claim) + +rc = subprocess.call(sys.argv[1:]) + +# Return all the reserved slots. +if len(jobs): + os.write(writer, jobs) + +sys.exit(rc) -- cgit v1.2.3