From aca146113515646934f50f3f278134b6eb89d60c Mon Sep 17 00:00:00 2001 From: Thomas Nagy Date: Sat, 31 Oct 2015 14:26:13 +0100 Subject: [PATCH 1/3] thirdparty:waf: Update gccdeps from upstream The upstream version provides a backport from waf 1.8 that features flag detection and more robust dependency processing (missing dependencies will raise errors). Signed-off-by: Thomas Nagy --- third_party/waf/wafadmin/3rdparty/gccdeps.py | 212 ++++++++++++++++++++------- 1 file changed, 156 insertions(+), 56 deletions(-) diff --git a/third_party/waf/wafadmin/3rdparty/gccdeps.py b/third_party/waf/wafadmin/3rdparty/gccdeps.py index 28a889d..3c53b16 100644 --- a/third_party/waf/wafadmin/3rdparty/gccdeps.py +++ b/third_party/waf/wafadmin/3rdparty/gccdeps.py @@ -4,113 +4,175 @@ """ Execute the tasks with gcc -MD, read the dependencies from the .d file -and prepare the dependency calculation for the next run +and prepare the dependency calculation for the next run. + +Usage: + def configure(conf): + conf.load('gccdeps') """ import os, re, threading -import Task, Logs, Utils, preproc -from TaskGen import before, after, feature +import Options, Task, Logs, Utils, Constants, preproc +from TaskGen import before, feature lock = threading.Lock() -preprocessor_flag = '-MD' +gccdeps_flags = ['-MD'] +if not preproc.go_absolute: + gccdeps_flags = ['-MMD'] -@feature('cc') -@before('apply_core') -def add_mmd_cc(self): - if self.env.get_flat('CCFLAGS').find(preprocessor_flag) < 0: - self.env.append_value('CCFLAGS', preprocessor_flag) - -@feature('cxx') -@before('apply_core') -def add_mmd_cxx(self): - if self.env.get_flat('CXXFLAGS').find(preprocessor_flag) < 0: - self.env.append_value('CXXFLAGS', preprocessor_flag) +# Third-party tools are allowed to add extra names in here with append() +supported_compilers = ['gcc', 'icc', 'clang'] def scan(self): - "the scanner does not do anything initially" + if not self.__class__.__name__ in self.env.ENABLE_GCCDEPS: + if not self.env.GCCDEPS: + raise Utils.WafError('Load gccdeps in configure!') + return self.no_gccdeps_scan() nodes = self.generator.bld.node_deps.get(self.unique_id(), []) names = [] return (nodes, names) re_o = re.compile("\.o$") -re_src = re.compile("^(\.\.)[\\/](.*)$") +re_splitter = re.compile(r'(?= 0: + return line[sep_idx + 2:] + else: + return line + +def path_to_node(base_node, path, cached_nodes): + # Take the base node and the path and return a node + # Results are cached because searching the node tree is expensive + # The following code is executed by threads, it is not safe, so a lock is needed... + if getattr(path, '__hash__'): + node_lookup_key = (id(base_node), path) + else: + # Not hashable, assume it is a list and join into a string + node_lookup_key = (id(base_node), os.path.sep.join(path)) + try: + lock.acquire() + node = cached_nodes[node_lookup_key] + except KeyError: + node = base_node.find_resource(path) + cached_nodes[node_lookup_key] = node + finally: + lock.release() + return node def post_run(self): # The following code is executed by threads, it is not safe, so a lock is needed... + if not self.__class__.__name__ in self.env.ENABLE_GCCDEPS: + return self.no_gccdeps_post_run() + if getattr(self, 'cached', None): return Task.Task.post_run(self) name = self.outputs[0].abspath(self.env) name = re_o.sub('.d', name) txt = Utils.readf(name) - #os.unlink(name) - + #os.remove(name) + + # Compilers have the choice to either output the file's dependencies + # as one large Makefile rule: + # + # /path/to/file.o: /path/to/dep1.h \ + # /path/to/dep2.h \ + # /path/to/dep3.h \ + # ... + # + # or as many individual rules: + # + # /path/to/file.o: /path/to/dep1.h + # /path/to/file.o: /path/to/dep2.h + # /path/to/file.o: /path/to/dep3.h + # ... + # + # So the first step is to sanitize the input by stripping out the left- + # hand side of all these lines. After that, whatever remains are the + # implicit dependencies of task.outputs[0] + txt = '\n'.join([remove_makefile_rule_lhs(line) for line in txt.splitlines()]) + + # Now join all the lines together txt = txt.replace('\\\n', '') - lst = txt.strip().split(':') - val = ":".join(lst[1:]) - val = val.split() + val = txt.strip() + val = [x.replace('\\ ', ' ') for x in re_splitter.split(val) if x] nodes = [] bld = self.generator.bld + variant = self.env.variant() + + # Dynamically bind to the cache + try: + cached_nodes = bld.cached_nodes + except AttributeError: + cached_nodes = bld.cached_nodes = {} - f = re.compile("^("+self.env.variant()+"|\.\.)[\\/](.*)$") for x in val: - if os.path.isabs(x): + node = None + if os.path.isabs(x): if not preproc.go_absolute: continue - - lock.acquire() - try: - node = bld.root.find_resource(x) - finally: - lock.release() + node = path_to_node(bld.root, x, cached_nodes) else: - g = re.search(re_src, x) - if g: - x = g.group(2) - lock.acquire() - try: - node = bld.bldnode.parent.find_resource(x) - finally: - lock.release() + # when calling find_resource, make sure the path does not contain '..' + x = [k for k in Utils.split_path(x) if k and k != '.'] + + + level = 0 + while '..' in x: + idx = x.index('..') + if idx == 0: + x = x[1:] + level += 1 + else: + del x[idx] + del x[idx-1] + + path = bld.bldnode + if x and x[0] == variant: + x = x[1:] + path = bld.srcnode else: - g = re.search(f, x) - if g: - x = g.group(2) - lock.acquire() - try: - node = bld.srcnode.find_resource(x) - finally: - lock.release() + while level: + path = path.parent + level -= 1 + node = path_to_node(path, x, cached_nodes) + + if not node: + raise ValueError('could not find %r for %r' % (x, self)) if id(node) == id(self.inputs[0]): # ignore the source file, it is already in the dependencies # this way, successful config tests may be retrieved from the cache continue + nodes.append(node) - if not node: - raise ValueError('could not find %r for %r' % (x, self)) - else: - nodes.append(node) - - Logs.debug('deps: real scanner for %s returned %s' % (str(self), str(nodes))) - + Logs.debug('deps: gccdeps for %s returned %s' % (str(self), str(nodes))) bld.node_deps[self.unique_id()] = nodes bld.raw_deps[self.unique_id()] = [] try: del self.cache_sig - except: + except AttributeError: pass Task.Task.post_run(self) -import Constants, Utils def sig_implicit_deps(self): + if not self.__class__.__name__ in self.env.ENABLE_GCCDEPS: + return self.no_gccdeps_sig_implicit_deps() try: return Task.Task.sig_implicit_deps(self) except Utils.WafError: @@ -122,6 +184,44 @@ def sig_implicit_deps(self): except KeyError: pass else: - cls.post_run = post_run + cls.no_gccdeps_scan = cls.scan + cls.no_gccdeps_post_run = cls.post_run + cls.no_gccdeps_sig_implicit_deps = cls.sig_implicit_deps + cls.scan = scan + cls.post_run = post_run cls.sig_implicit_deps = sig_implicit_deps + +@before('apply_core') +@feature('force_gccdeps') +def force_gccdeps(self): + self.env.ENABLE_GCCDEPS = ['cc', 'cxx'] + +def detect(conf): + # record that the configuration was executed properly + conf.env.GCCDEPS = True + + # in case someone provides a --enable-gccdeps or --disable-gccdeps command-line option + if not getattr(Options.options, 'enable_gccdeps', True): + return + + global gccdeps_flags + flags = conf.env.GCCDEPS_FLAGS or gccdeps_flags + if conf.env.CC_NAME in supported_compilers: + try: + conf.check(fragment='int main() { return 0; }', features='c force_gccdeps', ccflags=flags, msg='Checking for c flags %r' % ''.join(flags)) + except Utils.WafError: + pass + else: + conf.env.append_value('CCFLAGS', gccdeps_flags) + conf.env.append_unique('ENABLE_GCCDEPS', 'cc') + + if conf.env.CXX_NAME in supported_compilers: + try: + conf.check(fragment='int main() { return 0; }', features='cxx force_gccdeps', cxxflags=flags, msg='Checking for cxx flags %r' % ''.join(flags)) + except Utils.WafError: + pass + else: + conf.env.append_value('CXXFLAGS', gccdeps_flags) + conf.env.append_unique('ENABLE_GCCDEPS', 'cxx') + From c037a456db9d10b9b2318e0e65cad53b9eac16fd Mon Sep 17 00:00:00 2001 From: Thomas Nagy Date: Mon, 16 Nov 2015 23:45:55 +0100 Subject: [PATCH 2/3] build:wafsamba: Use the upstream version of gccdeps This removes the duplicate gccdeps file provided in the Samba tree. The newer gccdeps extension will use the -MMD flag instead of -MD as dependencies on system headers are ignored by default. Signed-off-by: Thomas Nagy --- buildtools/wafsamba/wscript | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/buildtools/wafsamba/wscript b/buildtools/wafsamba/wscript index a4cb620..fa4be9b 100755 --- a/buildtools/wafsamba/wscript +++ b/buildtools/wafsamba/wscript @@ -226,30 +226,7 @@ def configure(conf): # we need git for 'waf dist' conf.find_program('git', var='GIT') - - # older gcc versions (< 4.4) does not work with gccdeps, so we have to see if the .d file is generated - if Options.options.enable_gccdeps: - from TaskGen import feature, after - @feature('testd') - @after('apply_core') - def check_d(self): - tsk = self.compiled_tasks[0] - tsk.outputs.append(tsk.outputs[0].change_ext('.d')) - - import Task - cc = Task.TaskBase.classes['cc'] - oldmeth = cc.run - - cc.run = Task.compile_fun_noshell('cc', '${CC} ${CCFLAGS} ${CPPFLAGS} ${_CCINCFLAGS} ${_CCDEFFLAGS} ${CC_SRC_F}${SRC} ${CC_TGT_F}${TGT[0].abspath(env)}')[0] - try: - try: - conf.check(features='c testd', fragment='int main() {return 0;}\n', ccflags=['-MD'], mandatory=True, msg='Check for -MD') - except: - pass - else: - conf.check_tool('gccdeps', tooldir=conf.srcdir + "/buildtools/wafsamba") - finally: - cc.run = oldmeth + conf.check_tool('gccdeps') # make the install paths available in environment conf.env.LIBDIR = Options.options.LIBDIR or '${PREFIX}/lib' @@ -572,6 +549,13 @@ struct foo bar = { .y = 'X', .x = 1 }; def build(bld): + # the file gccdeps.pyc is not removed after "git pull", and a reconfiguration is necessary + p = os.path.join(bld.curdir, 'gccdeps.pyc') + if 'gccdeps' in sys.modules and (os.path.exists(p) or not bld.env.GCCDEPS): + if os.path.exists(p): + os.remove(p) + raise Utils.WafError('The gccdeps module has been upgraded. Please run distclean and reconfigure') + # give a more useful message if the source directory has moved relpath = os_path_relpath(bld.curdir, bld.srcnode.abspath()) if relpath.find('../') != -1: From 708cbbdbf8972555709739ff64634d5c0c684b47 Mon Sep 17 00:00:00 2001 From: Thomas Nagy Date: Sat, 31 Oct 2015 14:32:22 +0100 Subject: [PATCH 3/3] build:wafsamba: Change --enable-gccdeps to --disable-gccdeps The gccdeps option is always enable so the option has no effect. This change turns --enable-gccdeps to --disable-gccdeps. Signed-off-by: Thomas Nagy --- buildtools/wafsamba/wscript | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildtools/wafsamba/wscript b/buildtools/wafsamba/wscript index fa4be9b..34775a6 100755 --- a/buildtools/wafsamba/wscript +++ b/buildtools/wafsamba/wscript @@ -106,9 +106,9 @@ def set_options(opt): gr.add_option('--fatal-errors', help=("Stop compilation on first error (enable -Wfatal-errors)"), action="store_true", dest='fatal_errors', default=False) - gr.add_option('--enable-gccdeps', - help=("Enable use of gcc -MD dependency module"), - action="store_true", dest='enable_gccdeps', default=True) + gr.add_option('--disable-gccdeps', + help="Disable use of gcc -MD dependency module", + action="store_false", dest='enable_gccdeps', default=True) gr.add_option('--timestamp-dependencies', help=("use file timestamps instead of content for build dependencies (BROKEN)"), action="store_true", dest='timestamp_dependencies', default=False)