From b581f93fdc782120661131876fb49c1686317051 Mon Sep 17 00:00:00 2001 From: Thomas Nagy Date: Sat, 31 Oct 2015 14:26:13 +0100 Subject: [PATCH 1/3] build:wafsamba: Update gccdeps from upstream The upstream version provides a backport from waf 1.8 that features flag detection and more robust dependency processing (missing dependencies will raise errors). Signed-off-by: Thomas Nagy --- third_party/waf/wafadmin/3rdparty/gccdeps.py | 212 ++++++++++++++++++++------- 1 file changed, 156 insertions(+), 56 deletions(-) diff --git a/third_party/waf/wafadmin/3rdparty/gccdeps.py b/third_party/waf/wafadmin/3rdparty/gccdeps.py index 28a889d..3c53b16 100644 --- a/third_party/waf/wafadmin/3rdparty/gccdeps.py +++ b/third_party/waf/wafadmin/3rdparty/gccdeps.py @@ -4,113 +4,175 @@ """ Execute the tasks with gcc -MD, read the dependencies from the .d file -and prepare the dependency calculation for the next run +and prepare the dependency calculation for the next run. + +Usage: + def configure(conf): + conf.load('gccdeps') """ import os, re, threading -import Task, Logs, Utils, preproc -from TaskGen import before, after, feature +import Options, Task, Logs, Utils, Constants, preproc +from TaskGen import before, feature lock = threading.Lock() -preprocessor_flag = '-MD' +gccdeps_flags = ['-MD'] +if not preproc.go_absolute: + gccdeps_flags = ['-MMD'] -@feature('cc') -@before('apply_core') -def add_mmd_cc(self): - if self.env.get_flat('CCFLAGS').find(preprocessor_flag) < 0: - self.env.append_value('CCFLAGS', preprocessor_flag) - -@feature('cxx') -@before('apply_core') -def add_mmd_cxx(self): - if self.env.get_flat('CXXFLAGS').find(preprocessor_flag) < 0: - self.env.append_value('CXXFLAGS', preprocessor_flag) +# Third-party tools are allowed to add extra names in here with append() +supported_compilers = ['gcc', 'icc', 'clang'] def scan(self): - "the scanner does not do anything initially" + if not self.__class__.__name__ in self.env.ENABLE_GCCDEPS: + if not self.env.GCCDEPS: + raise Utils.WafError('Load gccdeps in configure!') + return self.no_gccdeps_scan() nodes = self.generator.bld.node_deps.get(self.unique_id(), []) names = [] return (nodes, names) re_o = re.compile("\.o$") -re_src = re.compile("^(\.\.)[\\/](.*)$") +re_splitter = re.compile(r'(?= 0: + return line[sep_idx + 2:] + else: + return line + +def path_to_node(base_node, path, cached_nodes): + # Take the base node and the path and return a node + # Results are cached because searching the node tree is expensive + # The following code is executed by threads, it is not safe, so a lock is needed... + if getattr(path, '__hash__'): + node_lookup_key = (id(base_node), path) + else: + # Not hashable, assume it is a list and join into a string + node_lookup_key = (id(base_node), os.path.sep.join(path)) + try: + lock.acquire() + node = cached_nodes[node_lookup_key] + except KeyError: + node = base_node.find_resource(path) + cached_nodes[node_lookup_key] = node + finally: + lock.release() + return node def post_run(self): # The following code is executed by threads, it is not safe, so a lock is needed... + if not self.__class__.__name__ in self.env.ENABLE_GCCDEPS: + return self.no_gccdeps_post_run() + if getattr(self, 'cached', None): return Task.Task.post_run(self) name = self.outputs[0].abspath(self.env) name = re_o.sub('.d', name) txt = Utils.readf(name) - #os.unlink(name) - + #os.remove(name) + + # Compilers have the choice to either output the file's dependencies + # as one large Makefile rule: + # + # /path/to/file.o: /path/to/dep1.h \ + # /path/to/dep2.h \ + # /path/to/dep3.h \ + # ... + # + # or as many individual rules: + # + # /path/to/file.o: /path/to/dep1.h + # /path/to/file.o: /path/to/dep2.h + # /path/to/file.o: /path/to/dep3.h + # ... + # + # So the first step is to sanitize the input by stripping out the left- + # hand side of all these lines. After that, whatever remains are the + # implicit dependencies of task.outputs[0] + txt = '\n'.join([remove_makefile_rule_lhs(line) for line in txt.splitlines()]) + + # Now join all the lines together txt = txt.replace('\\\n', '') - lst = txt.strip().split(':') - val = ":".join(lst[1:]) - val = val.split() + val = txt.strip() + val = [x.replace('\\ ', ' ') for x in re_splitter.split(val) if x] nodes = [] bld = self.generator.bld + variant = self.env.variant() + + # Dynamically bind to the cache + try: + cached_nodes = bld.cached_nodes + except AttributeError: + cached_nodes = bld.cached_nodes = {} - f = re.compile("^("+self.env.variant()+"|\.\.)[\\/](.*)$") for x in val: - if os.path.isabs(x): + node = None + if os.path.isabs(x): if not preproc.go_absolute: continue - - lock.acquire() - try: - node = bld.root.find_resource(x) - finally: - lock.release() + node = path_to_node(bld.root, x, cached_nodes) else: - g = re.search(re_src, x) - if g: - x = g.group(2) - lock.acquire() - try: - node = bld.bldnode.parent.find_resource(x) - finally: - lock.release() + # when calling find_resource, make sure the path does not contain '..' + x = [k for k in Utils.split_path(x) if k and k != '.'] + + + level = 0 + while '..' in x: + idx = x.index('..') + if idx == 0: + x = x[1:] + level += 1 + else: + del x[idx] + del x[idx-1] + + path = bld.bldnode + if x and x[0] == variant: + x = x[1:] + path = bld.srcnode else: - g = re.search(f, x) - if g: - x = g.group(2) - lock.acquire() - try: - node = bld.srcnode.find_resource(x) - finally: - lock.release() + while level: + path = path.parent + level -= 1 + node = path_to_node(path, x, cached_nodes) + + if not node: + raise ValueError('could not find %r for %r' % (x, self)) if id(node) == id(self.inputs[0]): # ignore the source file, it is already in the dependencies # this way, successful config tests may be retrieved from the cache continue + nodes.append(node) - if not node: - raise ValueError('could not find %r for %r' % (x, self)) - else: - nodes.append(node) - - Logs.debug('deps: real scanner for %s returned %s' % (str(self), str(nodes))) - + Logs.debug('deps: gccdeps for %s returned %s' % (str(self), str(nodes))) bld.node_deps[self.unique_id()] = nodes bld.raw_deps[self.unique_id()] = [] try: del self.cache_sig - except: + except AttributeError: pass Task.Task.post_run(self) -import Constants, Utils def sig_implicit_deps(self): + if not self.__class__.__name__ in self.env.ENABLE_GCCDEPS: + return self.no_gccdeps_sig_implicit_deps() try: return Task.Task.sig_implicit_deps(self) except Utils.WafError: @@ -122,6 +184,44 @@ def sig_implicit_deps(self): except KeyError: pass else: - cls.post_run = post_run + cls.no_gccdeps_scan = cls.scan + cls.no_gccdeps_post_run = cls.post_run + cls.no_gccdeps_sig_implicit_deps = cls.sig_implicit_deps + cls.scan = scan + cls.post_run = post_run cls.sig_implicit_deps = sig_implicit_deps + +@before('apply_core') +@feature('force_gccdeps') +def force_gccdeps(self): + self.env.ENABLE_GCCDEPS = ['cc', 'cxx'] + +def detect(conf): + # record that the configuration was executed properly + conf.env.GCCDEPS = True + + # in case someone provides a --enable-gccdeps or --disable-gccdeps command-line option + if not getattr(Options.options, 'enable_gccdeps', True): + return + + global gccdeps_flags + flags = conf.env.GCCDEPS_FLAGS or gccdeps_flags + if conf.env.CC_NAME in supported_compilers: + try: + conf.check(fragment='int main() { return 0; }', features='c force_gccdeps', ccflags=flags, msg='Checking for c flags %r' % ''.join(flags)) + except Utils.WafError: + pass + else: + conf.env.append_value('CCFLAGS', gccdeps_flags) + conf.env.append_unique('ENABLE_GCCDEPS', 'cc') + + if conf.env.CXX_NAME in supported_compilers: + try: + conf.check(fragment='int main() { return 0; }', features='cxx force_gccdeps', cxxflags=flags, msg='Checking for cxx flags %r' % ''.join(flags)) + except Utils.WafError: + pass + else: + conf.env.append_value('CXXFLAGS', gccdeps_flags) + conf.env.append_unique('ENABLE_GCCDEPS', 'cxx') + From 097bc83bbd300f1af2bff0a53685b5ec4d6b9534 Mon Sep 17 00:00:00 2001 From: Thomas Nagy Date: Sat, 31 Oct 2015 14:29:54 +0100 Subject: [PATCH 2/3] build:wafsamba: Use the upstream version of gccdeps This removes the duplicate gccdeps file provided in the Samba tree. The newer gccdeps extension will use the -MMD flag instead of -MD as dependencies on system headers are ignored by default. Signed-off-by: Thomas Nagy --- buildtools/wafsamba/gccdeps.py | 127 ----------------------------------------- buildtools/wafsamba/wscript | 25 +------- 2 files changed, 1 insertion(+), 151 deletions(-) delete mode 100644 buildtools/wafsamba/gccdeps.py diff --git a/buildtools/wafsamba/gccdeps.py b/buildtools/wafsamba/gccdeps.py deleted file mode 100644 index 47505f0..0000000 --- a/buildtools/wafsamba/gccdeps.py +++ /dev/null @@ -1,127 +0,0 @@ -# encoding: utf-8 -# Thomas Nagy, 2008-2010 (ita) - -""" -Execute the tasks with gcc -MD, read the dependencies from the .d file -and prepare the dependency calculation for the next run -""" - -import os, re, threading -import Task, Logs, Utils, preproc -from TaskGen import before, after, feature - -lock = threading.Lock() - -preprocessor_flag = '-MD' - -@feature('c', 'cc') -@before('apply_core') -def add_mmd_cc(self): - if self.env.get_flat('CCFLAGS').find(preprocessor_flag) < 0: - self.env.append_value('CCFLAGS', preprocessor_flag) - -@feature('cxx') -@before('apply_core') -def add_mmd_cxx(self): - if self.env.get_flat('CXXFLAGS').find(preprocessor_flag) < 0: - self.env.append_value('CXXFLAGS', preprocessor_flag) - -def scan(self): - "the scanner does not do anything initially" - nodes = self.generator.bld.node_deps.get(self.unique_id(), []) - names = [] - return (nodes, names) - -re_o = re.compile("\.o$") -re_src = re.compile("^(\.\.)[\\/](.*)$") - -def post_run(self): - # The following code is executed by threads, it is not safe, so a lock is needed... - - if getattr(self, 'cached', None): - return Task.Task.post_run(self) - - name = self.outputs[0].abspath(self.env) - name = re_o.sub('.d', name) - txt = Utils.readf(name) - #os.unlink(name) - - txt = txt.replace('\\\n', '') - - lst = txt.strip().split(':') - val = ":".join(lst[1:]) - val = val.split() - - nodes = [] - bld = self.generator.bld - - f = re.compile("^("+self.env.variant()+"|\.\.)[\\/](.*)$") - for x in val: - if os.path.isabs(x): - - if not preproc.go_absolute: - continue - - lock.acquire() - try: - node = bld.root.find_resource(x) - finally: - lock.release() - else: - g = re.search(re_src, x) - if g: - x = g.group(2) - lock.acquire() - try: - node = bld.bldnode.parent.find_resource(x) - finally: - lock.release() - else: - g = re.search(f, x) - if g: - x = g.group(2) - lock.acquire() - try: - node = bld.srcnode.find_resource(x) - finally: - lock.release() - - if id(node) == id(self.inputs[0]): - # ignore the source file, it is already in the dependencies - # this way, successful config tests may be retrieved from the cache - continue - - if not node: - raise ValueError('could not find %r for %r' % (x, self)) - else: - nodes.append(node) - - Logs.debug('deps: real scanner for %s returned %s' % (str(self), str(nodes))) - - bld.node_deps[self.unique_id()] = nodes - bld.raw_deps[self.unique_id()] = [] - - try: - del self.cache_sig - except: - pass - - Task.Task.post_run(self) - -import Constants, Utils -def sig_implicit_deps(self): - try: - return Task.Task.sig_implicit_deps(self) - except Utils.WafError: - return Constants.SIG_NIL - -for name in 'cc cxx'.split(): - try: - cls = Task.TaskBase.classes[name] - except KeyError: - pass - else: - cls.post_run = post_run - cls.scan = scan - cls.sig_implicit_deps = sig_implicit_deps - diff --git a/buildtools/wafsamba/wscript b/buildtools/wafsamba/wscript index 69c556c..be4ab8b 100755 --- a/buildtools/wafsamba/wscript +++ b/buildtools/wafsamba/wscript @@ -225,30 +225,7 @@ def configure(conf): # we need git for 'waf dist' conf.find_program('git', var='GIT') - - # older gcc versions (< 4.4) does not work with gccdeps, so we have to see if the .d file is generated - if Options.options.enable_gccdeps: - from TaskGen import feature, after - @feature('testd') - @after('apply_core') - def check_d(self): - tsk = self.compiled_tasks[0] - tsk.outputs.append(tsk.outputs[0].change_ext('.d')) - - import Task - cc = Task.TaskBase.classes['cc'] - oldmeth = cc.run - - cc.run = Task.compile_fun_noshell('cc', '${CC} ${CCFLAGS} ${CPPFLAGS} ${_CCINCFLAGS} ${_CCDEFFLAGS} ${CC_SRC_F}${SRC} ${CC_TGT_F}${TGT[0].abspath(env)}')[0] - try: - try: - conf.check(features='c testd', fragment='int main() {return 0;}\n', ccflags=['-MD'], mandatory=True, msg='Check for -MD') - except: - pass - else: - conf.check_tool('gccdeps', tooldir=conf.srcdir + "/buildtools/wafsamba") - finally: - cc.run = oldmeth + conf.check_tool('gccdeps') # make the install paths available in environment conf.env.LIBDIR = Options.options.LIBDIR or '${PREFIX}/lib' From a1fabe2e83a2d54bbc36019c9cea89401fda30bd Mon Sep 17 00:00:00 2001 From: Thomas Nagy Date: Sat, 31 Oct 2015 14:32:22 +0100 Subject: [PATCH 3/3] build:wafsamba: Change --enable-gccdeps to --disable-gccdeps The gccdeps option is always enable so the option has no effect. This change turns --enable-gccdeps to --disable-gccdeps. Signed-off-by: Thomas Nagy --- buildtools/wafsamba/wscript | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildtools/wafsamba/wscript b/buildtools/wafsamba/wscript index be4ab8b..0cae0b2 100755 --- a/buildtools/wafsamba/wscript +++ b/buildtools/wafsamba/wscript @@ -106,9 +106,9 @@ def set_options(opt): gr.add_option('--fatal-errors', help=("Stop compilation on first error (enable -Wfatal-errors)"), action="store_true", dest='fatal_errors', default=False) - gr.add_option('--enable-gccdeps', - help=("Enable use of gcc -MD dependency module"), - action="store_true", dest='enable_gccdeps', default=True) + gr.add_option('--disable-gccdeps', + help="Disable use of gcc -MD dependency module", + action="store_false", dest='enable_gccdeps', default=True) gr.add_option('--timestamp-dependencies', help=("use file timestamps instead of content for build dependencies (BROKEN)"), action="store_true", dest='timestamp_dependencies', default=False)