Exctracting source code from EXAMPLES

Gidday!

I have ready some code to extract source code from EXAMPLES in man-pages. For that, I set up some convention:

Enclose the code (including the enclosing .EX/.EE in a pair of comments with a very precise formatting:

[[
...
.\" SRC BEGIN (program_name.c)
.EX
#include <stdio.h>

int main(void)

{
	printf("Hello, world!");
}

.EE
.\" SRC END
...
]]

There can be multiple programs in a single page, with the only restriction that each of them has to have a different program_name (there can be collisions within different manual pages, but not within the same manual page)

The Makefile will create a directory for each manal page, where the different programs will be created with the name specified in the comment (that's why it has to be different from others in the same page only).

Please, check that you like what you see, and comment if not (or if yes too :).

I tested it with membarrier.2, and it produced a correct .c file. The next step will be to add targets to lint and compile the produced files, to check their correctness.

I hope this will make our lives much easier maintaining manual pages :-)

Cheers,

Alex

index 03ebde18c..05a1b5950 100644
@@ -30,19 +30,20 @@ MAKEFLAGS += --no-print-directory
 MAKEFLAGS += --warn-undefined-variables
 
 
-srcdir := .
+srcdir   := .
 builddir := tmp
-LINTDIR := $(builddir)/lint
-HTMLDIR := $(builddir)/html
+LINTDIR  := $(builddir)/lint
+HTMLDIR  := $(builddir)/html
+SRCDIR   := $(builddir)/src
 
 DESTDIR :=
-prefix := /usr/local
-SYSCONFDIR := $(srcdir)/etc
-TMACDIR := $(SYSCONFDIR)/groff/tmac
+prefix  := /usr/local
+SYSCONFDIR  := $(srcdir)/etc
+TMACDIR     := $(SYSCONFDIR)/groff/tmac
 datarootdir := $(prefix)/share
-docdir := $(datarootdir)/doc
-MANDIR := $(srcdir)
-mandir := $(datarootdir)/man
+docdir  := $(datarootdir)/doc
+MANDIR  := $(srcdir)
+mandir  := $(datarootdir)/man
 MAN0DIR := $(MANDIR)/man0
 MAN1DIR := $(MANDIR)/man1
 MAN2DIR := $(MANDIR)/man2

@@ -61,7 +62,7 @@ man5dir := $(mandir)/man5
 man6dir := $(mandir)/man6
 man7dir := $(mandir)/man7
 man8dir := $(mandir)/man8
-manext := \.[0-9]
+manext  := \.[0-9]
 man0ext := .0
 man1ext := .1
 man2ext := .2

@@ -71,9 +72,9 @@ man5ext := .5
 man6ext := .6
 man7ext := .7
 man8ext := .8
-htmldir := $(docdir)
+htmldir  := $(docdir)
 htmldir_ := $(htmldir)/man
-htmlext := .html
+htmlext  := .html
 
 TMACFILES            := $(sort $(shell find $(TMACDIR) -not -type d))
 TMACNAMES            := $(basename $(notdir $(TMACFILES)))

@@ -99,9 +100,11 @@ MAN2HTMLFLAGS         := $(DEFAULT_MAN2HTMLFLAGS)

$(EXTRA_MAN2HTMLFLAGS)
INSTALL := install
INSTALL_DATA := $(INSTALL) -m 644

 INSTALL_DIR  := $(INSTALL) -m 755 -d
+MKDIR        := mkdir -p
 RM           := rm
 RMDIR        := rmdir --ignore-fail-on-non-empty
 GROFF        := groff
+MAN          := man
 MANDOC       := mandoc
 MAN2HTML     := man2html

@@ -161,12 +164,14 @@ _man5pages := $(filter %$(man5ext),$(_manpages))
_man6pages := $(filter %$(man6ext),$(_manpages)) _man7pages := $(filter %$(man7ext),$(_manpages)) _man8pages := $(filter %$(man8ext),$(_manpages))
-LINT_groff := $(patsubst $(MANDIR)/%,$(LINTDIR)/%.lint.groff.touch,$(LINTPAGES)) -LINT_mandoc:= $(patsubst $(MANDIR)/%,$(LINTDIR)/%.lint.mandoc.touch,$(LINTPAGES)) +LINT_groff :=$(patsubst $(MANDIR)/%,$(LINTDIR)/%.lint.groff.touch,$(LINTPAGES)) +LINT_mandoc:=$(patsubst $(MANDIR)/%,$(LINTDIR)/%.lint.mandoc.touch,$(LINTPAGES)) +SRCPAGEDIRS:=$(patsubst $(MANDIR)/%,$(SRCDIR)/%,$(LINTPAGES))

 MANDIRS   := $(sort $(shell find $(MANDIR)/man? -type d))
 HTMLDIRS  := $(patsubst $(MANDIR)/%,$(HTMLDIR)/%/.,$(MANDIRS))
 LINTDIRS  := $(patsubst $(MANDIR)/%,$(LINTDIR)/%/.,$(MANDIRS))
+SRCDIRS   := $(patsubst $(MANDIR)/%,$(SRCDIR)/%/.,$(MANDIRS))

_htmldirs := $(patsubst $(HTMLDIR)/%,$(DESTDIR)$(htmldir_)/%,$(HTMLDIRS)) _mandirs := $(patsubst $(MANDIR)/%,$(DESTDIR)$(mandir)/%/.,$(MANDIRS)) _man0dir := $(filter %man0/.,$(_mandirs))
@@ -248,6 +253,37 @@ uninstall-man: $(_mandir_rmdir) $(uninstall_manX)

        @:

+########################################################################
+# src
+
+$(SRCPAGEDIRS): $(SRCDIR)/%: $(MANDIR)/% | $$(@D)/.

+       $(info MKDIR    $@ $<)
+       $(RM) -rf $@
+       $(MKDIR) [email protected]
+       <$< \
+       sed -n 's/\.\\" SRC BEGIN (\(.*.c\))/\1/p' \
+       | while read f; do \
+               <$< \
+               sed -n \
+                       -e '/^\.TH/,/^\.SH/{/^\.SH/!p}' \
+                       -e '/^\.SH EXAMPLES/p' \
+                       -e "/^\... SRC BEGIN ($$f)$$/,/^\... SRC END$$/p" \
+               | $(MAN) -P cat -l - \
+               | sed '/^[^ ]/d' \
+               >[email protected]/$$f; \
+       done \
+       || exit $$?
+       mv -T [email protected] $@

+
+.PHONY: build-src src
+build-src src: $(SRCPAGEDIRS) | builddirs-src
+ @:
+
+.PHONY: builddirs-src
+builddirs-src: $(SRCDIRS)
+ @:
+
+
########################################################################

 # lint

index b2e3e035e..a46283dd7 100644
@@ -319,6 +319,7 @@ following code (x86) can be transformed using
 .BR membarrier ():
 .PP
 .in +4n
+.\" SRC BEGIN (membarrier.c)
 .EX
 #include <stdlib.h>
 

@@ -365,6 +366,7 @@ main(int argc, char *argv[])
     exit(EXIT_SUCCESS);
 }
 .EE
+.\" SRC END
 .in
 .PP
 The code above transformed to use

Alejandro Colomar
Linux man-pages comaintainer; https://www.kernel.org/doc/man-pages/
http://www.alejandro-colomar.es/

Hi Alex,

Alejandro Colomar (man-pages) wrote on Sun, Mar 20, 2022 at 09:34:47PM +0100:

Frankly, i don't see the point at all.

Manual Pages are not HOWTO documents that mindless users are supposed to copy from verbatim without understanding what they see. Instead, the are supposed to be read with your brain switched on and the reader is supposed to *apply* what they learnt, not copy it.

Ugly as hell. I would very strongly object to have anything like that added to any manual pages i maintain. When people add comments in order to convey syntax and semantics to a machine, that is a sure sign that the design of whatever it is intended to achieve was totally botched.

If any code snippet from an EXAMPLES section does compile, i would argue that it is severely ill-designed as it obviously contains lots of needless fluff that distracts from the point the example is actually trying to demonstrate. It ought to be stripped down to what really matters, to become shorter, more readable, and more to the point.

Here are a few EXAMPLES sections (in formatted form for readability) that demonstate how EXAMPLES sections should look like:

EXAMPLES /* from chroot(2) */

     The following example changes the root directory to newroot,
     sets the current directory to the new root, and drops some
     setuid privileges.  There may be other privileges which need to
     be dropped as well.

           #include <err.h>
           #include <unistd.h>

           if (chroot(newroot) != 0 || chdir("/") != 0)
                   err(1, "%s", newroot);
           setresuid(getuid(), getuid(), getuid());

EXAMPLES /* from write(2) */

     A typical loop allowing partial writes looks like this:

     const char *buf;
     size_t bsz, off;
     ssize_t nw;
     int d;

     for (off = 0; off < bsz; off += nw)
             if ((nw = write(d, buf + off, bsz - off)) == 0 || nw == -1)
                     err(1, "write");

EXAMPLES /* from BIO_s_fd(3) */

     This is a file descriptor BIO version of "Hello World":

           BIO *out;
           out = BIO_new_fd(fileno(stdout), BIO_NOCLOSE);
           BIO_printf(out, "Hello World\n");
           BIO_free(out);

EXAMPLES /* from MB_CUR_MAX(3) */

     Size a buffer in a portable way to hold one single multibyte character:

           char     buf[MB_LEN_MAX];
           wchar_t  wchar;  /* input value */

           if (wctomb(buf, wchar) == -1)
                   /* error */

     Switch between code handling the ascii(7) and UTF-8 character
     encodings in an OpenBSD-specific way (not portable):

           if (MB_CUR_MAX == 1) {
                   /* Code to handle ASCII-encoded single-byte strings. */
           } else {
                   /* Code to handle UTF-8-encoded multibyte strings. */
           }

EXAMPLES /* from malloc(3) */

     If malloc() must be used with multiplication, be sure to test for
     overflow:

           size_t num, size;
           ...

           /* Check for size_t overflow */
           if (size && num > SIZE_MAX / size)
                   errc(1, EOVERFLOW, "overflow");

           if ((p = malloc(num * size)) == NULL)
                   err(1, NULL);

     The above test is not sufficient in all cases.  For example,
     multiplying ints requires a different set of checks:

           int num, size;
           ...

           /* Avoid invalid requests */
           if (size < 0 || num < 0)
                   errc(1, EOVERFLOW, "overflow");

           /* Check for signed int overflow */
           if (size && num > INT_MAX / size)
                   errc(1, EOVERFLOW, "overflow");

           if ((p = malloc(num * size)) == NULL)
                   err(1, NULL);

     Assuming the implementation checks for integer overflow as
     OpenBSD does, it is much easier to use calloc(), reallocarray(),
     or recallocarray().

     The above examples could be simplified to:

           if ((p = reallocarray(NULL, num, size)) == NULL)
                   err(1, NULL);

     or at the cost of initialization:

           if ((p = calloc(num, size)) == NULL)
                   err(1, NULL);

Yours,
Ingo

Hi Ingo,

This feature is not supposed to be used by manual page readers (users), but by manual page editors.

I wanted to do this a long time ago for a few reasons:

- I know some manual pages are incorrect in some minor ways, which I'd like to fix. For example, the includes in some examples are incorrect, typically with more than needed. Reading 100s of pages carefully to fix them would be impossible. And having slightly incorrect examples in the manual pages is something I would really like to avoid. We have received reports of programs having added headers because the manual page said they had to add them, but they were really not needed.

- Some times, incorrect patches accidentally break the manual page formatting (for example, "\n" instead of "\en"). If the patch is simple enough that I may not care enough to render the manual page to check its correctness, and I don't realize the small detail, I may apply a patch that breaks an example program very badly. Having a magic button that checks that the code at least compiles would greatly reduce those bugs.

- When reviewing an incoming patch, instead of me having to read through it and after some time then replying "hey, please render your manual pages before sending patches to see that your code doesn't produce what you thought", I can just run make build-src && make lint-src, and if it fails, I can tell the contributor: "run make build-src && make lint-src, and you'll notice that your program contains some serious problems. Rerun until you are happy with it. If you can't figure out how to fix something, you can ask.".

It's kind of a man-pages static analyzer. Having it passing is not a measure of how good a manual page is, but having it break is an indicator that the manual page may actually have some problems.

I first thought of some way to achieve this without markers, but the regex would be very unreliable with current manual pages. If we standardize them a bit more, these markers might be made unnecessary, but I'm not sure about that.

In this project, there are examples as the ones you point below, but they are usually embedded in the text. In the EXAMPLES section we usually have full programs, which are normally minimal working programs that demonstrate the functions described. I don't think they have much noise. I've sometimes used them myself, and I like them, because I have something working from which I can test stuff. Sometimes it's quite hard to translate man-pages text into a running program, and having a working example program helps disambiguate the text (at least in my brain it works like that).

[...]

Cheers,

Alex

Alejandro Colomar
Linux man-pages comaintainer; https://www.kernel.org/doc/man-pages/
http://www.alejandro-colomar.es/

[subject changed; [email protected] dropped from CC list since he's not quoted]

Indeed. For quite a while I've had medium-term plans to do something similar for the many examples in the groff Texinfo manual. I want to be sure they remain accurate.

Here's my sketch for whenever I get back around to this idea.

1. Extract examples with, e.g., 'sed -n '/^@Example/,/^@endExample/'.[1]
2. Use Texinfo comment lines within this example to:

2a. identify the example (an invisible figure caption, if you will);

and

2b. specify additional transformations that should take place on the
input and/or output--these are not necessarily just more 's' sed commands but possibly 'i' and 'd' commands as well.

As an example of an 'i' use case, the line length _has_ to be shortened for many examples to fit within DVI/PDF page margins. We note this in the front matter of the current version of the manual[1], but I'm uncertain of the utility of having it literally present in every case (particularly in examples related to issues conceptually distant from line length and/or breaking).

In some cases, either input or output is wholly elided when it sheds no further light.

I don't envision changing the manual generation process to _populate_ the examples--that should be done carefully and by hand by someone with a pedagogue's hat on. The idea is to _validate_ the examples, reading them in, performing a few invariant sed substitutions (the ones that distinguish input, output, and error streams in the example text), perform any further specified sed operations encoded as Texinfo comments, and then run groff and verify that they match.

Since it would run this way, I reckon it can become just another test in our suite. (For those who don't follow groff development, groff 1.22.4 shipped with 3 tests. groff 1.23 can be expected to have at least 111.)

Regards,
Branden

[1] These are custom Texinfo macros defined by the groff manual.
[2] https://git.savannah.gnu.org/cgit/groff.git/tree/doc/groff.texi#n901

Hi Alex,

I’ve been working on something similar, slightly further along (the linting targets work). The extraction scripts could do with some improvement, but the Makefile changes are small:

# Check that example programs include in man pages really build Makefile.examples: $(MANPAGES)
scripts/list-example-files $^ > $@

include Makefile.examples

# Sources are listed as well as objects to ensure we update all source files # CPPFLAGS and TARGET_ARCH are defined to avoid warnings
.PHONY: check-example-programs
check-example-programs: CFLAGS = -Wall
check-example-programs: CPPFLAGS =
check-example-programs: TARGET_ARCH =
check-example-programs: $(EXAMPLE_SRCS) $(EXAMPLE_OBJS)

.PHONY: clean-example-programs
clean-example-programs:

	rm -f $(EXAMPLE_SRCS) $(EXAMPLE_OBJS)

scripts/list-example-files builds a separate Makefile to extract all the programs, which ends up looking like

/home/steve/man-pages/man1/memusage.c: /home/steve/man-pages/man1/memusage.1
scripts/extract-example-files $<

/home/steve/man-pages/man1/prog.c /home/steve/man-pages/man1/libdemo.c: /home/steve/man-pages/man1/sprof.1
scripts/extract-example-files $<

The scripts are attached. The patterns used to identify source code are close to those already present: .EX/.EE introduced by “Program source” (in which case the source code is extracted to a C file named after the man page) or by a

\." Example file

comment which can optionally name the file.

This has identified some more man pages which need fixes to their example code, I’ll send patches tomorrow.

Regards,

Stephen

Hi Stephen!

I've been adding code for linting this evening too. Currently I'm compiling and linking the programs extracted. I only did it with membarrier.2 for now, and I've already fixed a line.

I also plan to add some static analyzers too, such as iwyu(1), clang-tidy(1), or checkpatch.pl and some others.

You could check my 'lint' branch here:
<http://www.alejandro-colomar.es/src/alx/linux/man-pages/man-pages.git/log/?h=lint>

Compare that with what you have, and we can develop some mix of both.

With what I have, you need to run:

make build-src && make build-ld

I like the idea of autogenerating and including a makefile, which allows listing the example programs in the process. In my current makefile, I need to run make build-src before any further actions on those files, since I don't know them at the time of setting variables.

I may take some bits from here.

Those scripts seem a bit messy, due to the problem of not having a standardized comment. Are they reliable? Or are there false negatives or positives?

Okay.

Cheers,

Alex

Alejandro Colomar
Linux man-pages comaintainer; https://www.kernel.org/doc/man-pages/
http://www.alejandro-colomar.es/

Hi Stephen,

Well, I managed to get everything within the Makefile, with some complex variable definition; no autogenerated makefiles, and no included makefiles. It makes the makefile a bit slow, however, by adding a fixed initialization overhead: make clean goes from 0.11 s to 1.96 s in my computer. 2 s is not too much for complex operations, but it's a bit nasty when you just want to clean, or you just changed one page. But I guess any solution will have a similar fixed overhead (or it will be fast, but will require a separate step such as make build-src).

But the result is quite neat and simple, compared to other options:

+SRCPAGEDIRS:=$(patsubst $(MANDIR)/%,$(SRCDIR)/%.d,$(LINTPAGES))
+UNITS_c := $(patsubst $(MANDIR)/%,$(SRCDIR)/%,$(shell \

+               find $(MANDIR)/man?/ -type f \
+               | grep '$(manext)$$' \
+               | while read m; do \

+ <$$m \
+ sed -n "s,^\.\\"'"'" SRC BEGIN
(\(.*.c\))$$,$$m.d\1,p";\
+ done))

+########################################################################
+# src
+
+$(SRCPAGEDIRS): $(SRCDIR)/%.d: $(MANDIR)/% | $$(@D)/.
+ $(info MKDIR $@)
+ $(MKDIR) $@
+
+$(UNITS_c): $$(@D)

+       $(info SED      $@)
+       <$(patsubst $(SRCDIR)/%.d,$(MANDIR)/%,$<) \
+       sed -n \
+               -e '/^\.TH/,/^\.SH/{/^\.SH/!p}' \
+               -e '/^\.SH EXAMPLES/p' \
+               -e "/^\... SRC BEGIN ($(@F))$$/,/^\... SRC END$$/p" \
+       | $(MAN) -P cat -l - \
+       | sed '/^[^ ]/d' \
+       | sed 's/^       //' \
+       >$@ \
+       || exit $$?

+
+$(SRCDIRS): %/.: | $$(dir %). $(SRCDIR)/.
+
+.PHONY: build-src src
+build-src src: $(UNITS_c) | builddirs-src
+ @:
+
+.PHONY: builddirs-src
+builddirs-src: $(SRCDIRS)
+ @:
+
+

Those two biggie snippets embedded into the Makefile are similar in essence to the 2 helper scripts you use. The benefit of embedding them in the Makefile is that I have full control of it, and can use variables directly from the Makefile. Also less files :).
And even though it's slower,
I prefer it over having to run make build-src manually.

I've updated my 'lint' branch.

Cheers,

Alex

Alejandro Colomar
Linux man-pages comaintainer; https://www.kernel.org/doc/man-pages/
http://www.alejandro-colomar.es/