Fix parallel build of the policy
PATCH 1/1 Fix parallel build of the policy
Before this commit, "make -j2" would execute twice at the same time the rules written to build tmp/all_post.conf because these rules were applied every time tmp/all_post.conf, tmp/all_attrs_types.conf and tmp/only_te_rules.conf needed to be built. However, executing twice in parallel such line is buggy:
$(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> \
tmpdir)/all_post.confThis is why "make" reports following error for parallel builds:
Compiling refpolicy-patched base module /usr/bin/checkmodule -M -U allow base.conf -o tmp/base.mod /usr/bin/checkmodule: loading policy configuration from base.conf policy/modules/kernel/ubac.te":710:ERROR 'syntax error' at token 'fs_use_trans' on line 26520:
fs_use_trans devtmpfs system_u:object_r:device_t:s0;
/usr/bin/checkmodule: error(s) encountered while parsing configuration
make: *** [tmp/base.mod] Error 1
This commit fixes this bug by splitting the rules in 3 different targets, in both monolithic and modular builds.
--- Rules.modular | 24 ++++++++++++++---------- Rules.monolithic | 24 ++++++++++++++---------- 2 files changed, 28 insertions(+), 20 deletions(-)
Rules.modular +14 -10
@@ -157,17 +157,21 @@ $(tmpdir)/post_te_files.conf: $(m4support) $(tmpdir)/generated_definitions.conf # extract attributes and put them first. extract post te stuff # like genfscon and put last. -$(tmpdir)/all_attrs_types.conf $(tmpdir)/only_te_rules.conf $(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf - $(verbose) $(get_type_attr_decl) $(tmpdir)/all_te_files.conf | $(SORT) > $(tmpdir)/all_attrs_types.conf - $(verbose) cat $(tmpdir)/post_te_files.conf > $(tmpdir)/all_post.conf +$(tmpdir)/all_attrs_types.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(get_type_attr_decl) $^ | $(SORT) > $@ + +$(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf + $(verbose) cat $(tmpdir)/post_te_files.conf > $@ # these have to run individually because order matters: - $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(comment_move_decl) $(tmpdir)/all_te_files.conf > $(tmpdir)/only_te_rules.conf + $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $@ || true + +$(tmpdir)/only_te_rules.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(comment_move_decl) $^ > $@ ######################################## #
Rules.monolithic +14 -10
@@ -144,17 +144,21 @@ $(tmpdir)/post_te_files.conf: $(m4support) $(tmpdir)/generated_definitions.conf # extract attributes and put them first. extract post te stuff # like genfscon and put last. -$(tmpdir)/all_attrs_types.conf $(tmpdir)/only_te_rules.conf $(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf - $(verbose) $(get_type_attr_decl) $(tmpdir)/all_te_files.conf | $(SORT) > $(tmpdir)/all_attrs_types.conf - $(verbose) cat $(tmpdir)/post_te_files.conf > $(tmpdir)/all_post.conf +$(tmpdir)/all_attrs_types.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(get_type_attr_decl) $^ | $(SORT) > $@ + +$(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf + $(verbose) cat $(tmpdir)/post_te_files.conf > $@ # these have to run individually because order matters: - $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(comment_move_decl) $(tmpdir)/all_te_files.conf > $(tmpdir)/only_te_rules.conf + $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $@ || true + +$(tmpdir)/only_te_rules.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(comment_move_decl) $^ > $@ ######################################## #
Signature · 1 line
1.8.5.4
PATCH 1/1 Fix parallel build of the policy
Quoted 17 lines · Nicolas Iooss
Before this commit, "make -j2" would execute twice at the same time the rules written to build
tmp/all_post.confbecause these rules were applied every timetmp/all_post.conf,tmp/all_attrs_types.confandtmp/only_te_rules.confneeded to be built. However, executing twice in parallel such line is buggy:$(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> \ tmpdir)/all_post.confThis is why "make" reports following error for parallel builds:
Compiling refpolicy-patched base module
/usr/bin/checkmodule-M -U allow base.conf -o tmp/base.mod/usr/bin/checkmodule: loading policy configuration from base.conf policy/modules/kernel/ubac.te":710:ERROR 'syntax error' at token 'fs_use_trans' on line 26520:
fs_use_transdevtmpfs system_u:object_r:device_t:s0;
Quoted 5 lines
/usr/bin/checkmodule: error(s) encountered while parsing configuration
make: *** [tmp/base.mod] Error 1This commit fixes this bug by splitting the rules in 3 different targets, in both monolithic and modular builds.
How much testing did you do to ensure there are no changes to the output files? It looks like its ok, but changing how the source files are constructed requires solid testing first.
Quoted 78 lines
--- Rules.modular | 24 ++++++++++++++---------- Rules.monolithic | 24 ++++++++++++++---------- 2 files changed, 28 insertions(+), 20 deletions(-)Rules.modular +14
-10@@ -157,17 +157,21 @@ $(tmpdir)/post_te_files.conf: $(m4support) $(tmpdir)/generated_definitions.conf # extract attributes and put them first. extract post te stuff # like genfscon and put last. -$(tmpdir)/all_attrs_types.conf $(tmpdir)/only_te_rules.conf $(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf - $(verbose) $(get_type_attr_decl) $(tmpdir)/all_te_files.conf | $(SORT) > $(tmpdir)/all_attrs_types.conf - $(verbose) cat $(tmpdir)/post_te_files.conf > $(tmpdir)/all_post.conf +$(tmpdir)/all_attrs_types.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(get_type_attr_decl) $^ | $(SORT) > $@ + +$(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf + $(verbose) cat $(tmpdir)/post_te_files.conf > $@ # these have to run individually because order matters: - $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(comment_move_decl) $(tmpdir)/all_te_files.conf > $(tmpdir)/only_te_rules.conf + $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $@ || true + +$(tmpdir)/only_te_rules.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(comment_move_decl) $^ > $@ ######################################## #Rules.monolithic +14
-10@@ -144,17 +144,21 @@ $(tmpdir)/post_te_files.conf: $(m4support) $(tmpdir)/generated_definitions.conf # extract attributes and put them first. extract post te stuff # like genfscon and put last. -$(tmpdir)/all_attrs_types.conf $(tmpdir)/only_te_rules.conf $(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf - $(verbose) $(get_type_attr_decl) $(tmpdir)/all_te_files.conf | $(SORT) > $(tmpdir)/all_attrs_types.conf - $(verbose) cat $(tmpdir)/post_te_files.conf > $(tmpdir)/all_post.conf +$(tmpdir)/all_attrs_types.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(get_type_attr_decl) $^ | $(SORT) > $@ + +$(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf + $(verbose) cat $(tmpdir)/post_te_files.conf > $@ # these have to run individually because order matters: - $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(comment_move_decl) $(tmpdir)/all_te_files.conf > $(tmpdir)/only_te_rules.conf + $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $@ || true + +$(tmpdir)/only_te_rules.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(comment_move_decl) $^ > $@ ######################################## #
Signature · 3 lines
Chris PeBenito Tresys Technology, LLC www.tresys.com | oss.tresys.com
PATCH 1/1 Fix parallel build of the policy
2014-03-06 14:41 GMT+01:00 Christopher J. PeBenito <[email protected]>:
Quoted 25 lines · Nicolas Iooss
Quoted 1 line
Before this commit, "make -j2" would execute twice at the same time the
rules
Quoted 1 line
written to build
tmp/all_post.confbecause these rules were appliedevery time
Quoted 1 line
tmp/all_post.conf,tmp/all_attrs_types.confandtmp/only_te_rules.confneeded
Quoted 7 lines
to be built. However, executing twice in parallel such line is buggy:
$(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> \ tmpdir)/all_post.confThis is why "make" reports following error for parallel builds:
Quoted 8 lines
Compiling refpolicy-patched base module
/usr/bin/checkmodule-M -U allow base.conf -o tmp/base.mod/usr/bin/checkmodule: loading policy configuration from base.conf policy/modules/kernel/ubac.te":710:ERROR 'syntax error' at token 'fs_use_trans' on line 26520:
fs_use_transdevtmpfs system_u:object_r:device_t:s0;
/usr/bin/checkmodule: error(s) encountered while parsingconfiguration
Quoted 2 lines
make: *** [tmp/base.mod] Error 1
Quoted 9 lines
Quoted 1 line
This commit fixes this bug by splitting the rules in 3 different
targets, in
Quoted 1 line
both monolithic and modular builds.
How much testing did you do to ensure there are no changes to the output files? It looks like its ok, but changing how the source files are constructed requires solid testing first.
I've written the attached shell script to check that the built and installed files are not changed by my patch. I've written how I used this script in a comment at the beginning of the file, in hope anyone can reproduce my tests and compare their results with mine.
On my system, "make bare && make -j4 conf" prints twice "Updating policy/modules.conf and policy/booleans.conf" and "python -E support/sedoctool.py -b policy/booleans.conf -m policy/modules.conf -x doc/policy.xml". The patch does not change anything about it. That's why I've used "make conf" and not "make -j$NJOBS conf" in my script.
Executing "make bare && make conf && make -j4 policy && make -j4 install install-appconfig install-headers install-docs install-src" is also kind of
broken: the content of /etc/selinux/refpolicy/src/policy can't be predicted after this. More precisely, I've seen
that /etc/selinux/refpolicy/src/policy/file_contexts and /etc/selinux/refpolicy/src/policy/homedir_template may not be installed when building the monolithic configuration. This is due to the fact that "install-src" target in Makefile hasn't got any dependency and hence the "cp -R . $(srcpath)/policy" command may be executed before all of the generated files are created. That's why "make install-src" is also on a distinct line in the script.
If that matters, I'm using make version 4.0.
Thanks
Nicolas
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://oss.tresys.com/pipermail/refpolicy/attachments/20140306/b3387206/attachment.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: test-refpolicy-patch.sh
Type: application/x-sh
Size: 3292 bytes
Desc: not available
Url : http://oss.tresys.com/pipermail/refpolicy/attachments/20140306/b3387206/attachment.sh
PATCH 1/1 Fix parallel build of the policy
In reply to Nicolas IoossQuoted 17 lines · Nicolas Iooss
Before this commit, "make -j2" would execute twice at the same time the rules written to build
tmp/all_post.confbecause these rules were applied every timetmp/all_post.conf,tmp/all_attrs_types.confandtmp/only_te_rules.confneeded to be built. However, executing twice in parallel such line is buggy:$(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> \ tmpdir)/all_post.confThis is why "make" reports following error for parallel builds:
Compiling refpolicy-patched base module
/usr/bin/checkmodule-M -U allow base.conf -o tmp/base.mod/usr/bin/checkmodule: loading policy configuration from base.conf policy/modules/kernel/ubac.te":710:ERROR 'syntax error' at token 'fs_use_trans' on line 26520:
fs_use_transdevtmpfs system_u:object_r:device_t:s0;
Quoted 5 lines
/usr/bin/checkmodule: error(s) encountered while parsing configuration
make: *** [tmp/base.mod] Error 1This commit fixes this bug by splitting the rules in 3 different targets, in both monolithic and modular builds.
Merged.
Quoted 78 lines
--- Rules.modular | 24 ++++++++++++++---------- Rules.monolithic | 24 ++++++++++++++---------- 2 files changed, 28 insertions(+), 20 deletions(-)Rules.modular +14
-10@@ -157,17 +157,21 @@ $(tmpdir)/post_te_files.conf: $(m4support) $(tmpdir)/generated_definitions.conf # extract attributes and put them first. extract post te stuff # like genfscon and put last. -$(tmpdir)/all_attrs_types.conf $(tmpdir)/only_te_rules.conf $(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf - $(verbose) $(get_type_attr_decl) $(tmpdir)/all_te_files.conf | $(SORT) > $(tmpdir)/all_attrs_types.conf - $(verbose) cat $(tmpdir)/post_te_files.conf > $(tmpdir)/all_post.conf +$(tmpdir)/all_attrs_types.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(get_type_attr_decl) $^ | $(SORT) > $@ + +$(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf + $(verbose) cat $(tmpdir)/post_te_files.conf > $@ # these have to run individually because order matters: - $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(comment_move_decl) $(tmpdir)/all_te_files.conf > $(tmpdir)/only_te_rules.conf + $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $@ || true + +$(tmpdir)/only_te_rules.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(comment_move_decl) $^ > $@ ######################################## #Rules.monolithic +14
-10@@ -144,17 +144,21 @@ $(tmpdir)/post_te_files.conf: $(m4support) $(tmpdir)/generated_definitions.conf # extract attributes and put them first. extract post te stuff # like genfscon and put last. -$(tmpdir)/all_attrs_types.conf $(tmpdir)/only_te_rules.conf $(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf - $(verbose) $(get_type_attr_decl) $(tmpdir)/all_te_files.conf | $(SORT) > $(tmpdir)/all_attrs_types.conf - $(verbose) cat $(tmpdir)/post_te_files.conf > $(tmpdir)/all_post.conf +$(tmpdir)/all_attrs_types.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(get_type_attr_decl) $^ | $(SORT) > $@ + +$(tmpdir)/all_post.conf: $(tmpdir)/all_te_files.conf $(tmpdir)/post_te_files.conf + $(verbose) cat $(tmpdir)/post_te_files.conf > $@ # these have to run individually because order matters: - $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $(tmpdir)/all_post.conf || true - $(verbose) $(comment_move_decl) $(tmpdir)/all_te_files.conf > $(tmpdir)/only_te_rules.conf + $(verbose) $(GREP) '^sid ' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) '^fs_use_(xattr|task|trans)' $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^genfscon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^portcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^netifcon $(tmpdir)/all_te_files.conf >> $@ || true + $(verbose) $(GREP) ^nodecon $(tmpdir)/all_te_files.conf >> $@ || true + +$(tmpdir)/only_te_rules.conf: $(tmpdir)/all_te_files.conf + $(verbose) $(comment_move_decl) $^ > $@ ######################################## #
Signature · 3 lines
Chris PeBenito Tresys Technology, LLC www.tresys.com | oss.tresys.com