# Params header.

  if 'SConscript' in files:
      build_dir = os.path.join(env['BUILDDIR'], root[len(base_dir) + 1:])

Alex Richardson has submitted this change. ( https://gem5-review.googlesource.com/c/public/gem5/+/68518?usp=email ) ( 6 is the latest approved patch-set. No files were changed between the latest approved patch-set and the submitted one. )Change subject: scons: allow building without duplicating source files ...................................................................... scons: allow building without duplicating source files This adds a new scons flag --no-duplicate-sources to build without linking source files to the build directory. I find this very helpful when using CLion, since I can now generate a compilation database using `bear scons build/ALL/gem5.debug --no-duplicate-sources` and CLion will now correctly semantically analyze all the files inside src/. It also ensures that clicking on a build warning/error now opens the real source file rather than a symlink. This is not enabled by default since it's possible that certain use cases are not working correctly, but the basic testing I've done so far appears to work just fine. It appears that with this change the `<root>/src` directory is no longer added to `PYTHONPATH` when running `tests/main.py`, so this change depends on https://gem5-review.git.corp.google.com/c/public/gem5/+/68757 Change-Id: Iddc9bf9c8211e68e5432c0a07f5c95f427c1ca16 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/68518 Tested-by: kokoro <noreply+kokoro@google.com> Reviewed-by: Bobby Bruce <bbruce@ucdavis.edu> Maintainer: Bobby Bruce <bbruce@ucdavis.edu> --- M SConstruct M ext/drampower/SConscript M ext/dramsim2/SConscript M ext/dramsim3/SConscript M ext/fputils/SConscript M ext/iostream3/SConscript M ext/libelf/SConscript M ext/libfdt/SConscript M ext/nomali/SConscript M ext/softfloat/SConscript M ext/systemc/SConscript M site_scons/gem5_scons/sources.py M src/SConscript M src/mem/slicc/symbols/SymbolTable.py M util/m5/SConstruct M util/statetrace/SConstruct M util/tlm/SConstruct 17 files changed, 91 insertions(+), 73 deletions(-) Approvals: Bobby Bruce: Looks good to me, approved; Looks good to me, approved kokoro: Regressions pass diff --git a/SConstruct b/SConstruct index 6abbb51..7d6f406 100755 --- a/SConstruct +++ b/SConstruct @@ -145,6 +145,9 @@ help='Enable support for the gprof profiler') AddOption('--pprof', action='store_true', help='Enable support for the pprof profiler') +AddOption('--no-duplicate-sources', action='store_false', + dest='duplicate_sources', + help='Do not create symlinks to sources in the build directory') # Inject the built_tools directory into the python path. sys.path[1:1] = [ Dir('#build_tools').abspath ] @@ -264,6 +267,7 @@ # Add shared top-level headers main.Prepend(CPPPATH=Dir('include')) +main.Prepend(CPPPATH=Dir('src')) ######################################################################## @@ -774,11 +778,13 @@ build_dir = os.path.relpath(root, ext_dir) SConscript(os.path.join(root, 'SConscript'), variant_dir=os.path.join(variant_ext, build_dir), - exports=exports) + exports=exports, + duplicate=GetOption('duplicate_sources')) # The src/SConscript file sets up the build rules in 'env' according # to the configured variables. It returns a list of environments, # one for each variant build (debug, opt, etc.) - SConscript('src/SConscript', variant_dir=variant_path, exports=exports) + SConscript('src/SConscript', variant_dir=variant_path, exports=exports, + duplicate=GetOption('duplicate_sources')) atexit.register(summarize_warnings) diff --git a/ext/drampower/SConscript b/ext/drampower/SConscript index 870d050..38acbf4 100644 --- a/ext/drampower/SConscript +++ b/ext/drampower/SConscript @@ -41,7 +41,7 @@ Import('env') -env.Prepend(CPPPATH=Dir('./src')) +env.Prepend(CPPPATH=Dir('./src').srcnode()) # Add the appropriate files for the library drampower_files = [] diff --git a/ext/dramsim2/SConscript b/ext/dramsim2/SConscript index 7eb178d..c296538 100644 --- a/ext/dramsim2/SConscript +++ b/ext/dramsim2/SConscript @@ -85,6 +85,6 @@ dramenv.Library('dramsim2', [dramenv.SharedObject(f) for f in dram_files]) -env.Prepend(CPPPATH=Dir('.')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['dramsim2']) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/dramsim3/SConscript b/ext/dramsim3/SConscript index b717816..6be9690 100644 --- a/ext/dramsim3/SConscript +++ b/ext/dramsim3/SConscript @@ -56,12 +56,12 @@ if thermal: superlu_path = os.path.join(dramsim_path, 'ext/SuperLU_MT_3.1/lib') - env.Prepend(CPPPATH=Dir('.')) + env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['dramsim3', 'superlu_mt_OPENMP', 'm', 'f77blas', 'atlas', 'gomp'], LIBPATH=[dramsim_path, superlu_path]) else: - env.Prepend(CPPPATH=Dir('.')) + env.Prepend(CPPPATH=Dir('.').srcnode()) # a littel hacky but can get a shared library working env.Append(LIBS=['dramsim3', 'gomp'], LIBPATH=[dramsim_path], # compile-time lookup diff --git a/ext/fputils/SConscript b/ext/fputils/SConscript index 6a8e44f..bc158c2 100644 --- a/ext/fputils/SConscript +++ b/ext/fputils/SConscript @@ -30,7 +30,7 @@ Import('env') -env.Prepend(CPPPATH=Dir('./include')) +env.Prepend(CPPPATH=Dir('./include').srcnode()) fpenv = env.Clone() diff --git a/ext/iostream3/SConscript b/ext/iostream3/SConscript index df0b213..3b4e937 100644 --- a/ext/iostream3/SConscript +++ b/ext/iostream3/SConscript @@ -41,6 +41,6 @@ env.Library('iostream3', [env.SharedObject('zfstream.cc')]) -env.Prepend(CPPPATH=Dir('.')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['iostream3']) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/libelf/SConscript b/ext/libelf/SConscript index 535e216..d6f8234 100644 --- a/ext/libelf/SConscript +++ b/ext/libelf/SConscript @@ -127,16 +127,19 @@ # Setup m4 tool m4env.Tool('m4') -m4env.Append(M4FLAGS=['-DSRCDIR=%s' % Dir('.').path]) +m4env.Append(M4FLAGS=['-DSRCDIR=%s' % Dir('.').srcnode().path]) m4env['M4COM'] = '$M4 $M4FLAGS $SOURCES > $TARGET' m4env.M4(target=File('libelf_convert.c'), - source=[File('elf_types.m4'), File('libelf_convert.m4')]) + source=[File('elf_types.m4').srcnode(), + File('libelf_convert.m4').srcnode()]) m4env.M4(target=File('libelf_fsize.c'), - source=[File('elf_types.m4'), File('libelf_fsize.m4')]) + source=[File('elf_types.m4').srcnode(), + File('libelf_fsize.m4').srcnode()]) m4env.M4(target=File('libelf_msize.c'), - source=[File('elf_types.m4'), File('libelf_msize.m4')]) + source=[File('elf_types.m4').srcnode(), + File('libelf_msize.m4').srcnode()]) -m4env.Append(CPPPATH=Dir('.')) +m4env.Append(CPPPATH=[Dir('.'), Dir('.').srcnode()]) # Build libelf as a static library with PIC code so it can be linked # into either m5 or the library @@ -146,6 +149,6 @@ m4env.Command(File('native-elf-format.h'), File('native-elf-format'), '${SOURCE} > ${TARGET}') -env.Prepend(CPPPATH=Dir('.')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=[File('libelf.a')]) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/libfdt/SConscript b/ext/libfdt/SConscript index 64573b7..a509bbe 100644 --- a/ext/libfdt/SConscript +++ b/ext/libfdt/SConscript @@ -44,6 +44,6 @@ FdtFile('fdt_strerror.c') env.Library('fdt', [env.SharedObject(f) for f in fdt_files]) -env.Prepend(CPPPATH=Dir('.')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['fdt']) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/nomali/SConscript b/ext/nomali/SConscript index b156ab0..bcc5cfb 100644 --- a/ext/nomali/SConscript +++ b/ext/nomali/SConscript @@ -39,7 +39,7 @@ Import('env') -env.Prepend(CPPPATH=Dir('./include')) +env.Prepend(CPPPATH=Dir('include').srcnode()) nomali = env.Clone() nomali.Append(CCFLAGS=['-Wno-ignored-qualifiers']) diff --git a/ext/softfloat/SConscript b/ext/softfloat/SConscript index b4a8d51..420a71e 100644 --- a/ext/softfloat/SConscript +++ b/ext/softfloat/SConscript @@ -420,6 +420,6 @@ sf_env.Library('softfloat', [sf_env.SharedObject(f) for f in softfloat_files]) -env.Prepend(CPPPATH=Dir('./')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['softfloat']) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/systemc/SConscript b/ext/systemc/SConscript index 89ef920..5248fc3 100644 --- a/ext/systemc/SConscript +++ b/ext/systemc/SConscript @@ -25,6 +25,7 @@ import os from m5.util.terminal import get_termcap +import gem5_scons Import('env') systemc = env.Clone() @@ -32,36 +33,30 @@ build_root = Dir('.').abspath src_root = Dir('.').srcdir.abspath -systemc.Prepend(CPPPATH=Dir('./src')) +systemc.Prepend(CPPPATH=Dir('./src').srcnode()) systemc.Prepend(CPATH=Dir('./src')) systemc.Prepend(CXXFLAGS=['-DSC_INCLUDE_FX']) systemc.Prepend(CFLAGS=['-DSC_INCLUDE_FX']) -conf = Configure(systemc, - conf_dir = os.path.join(build_root, '.scons_config'), - log_file = os.path.join(build_root, 'scons_config.log')) -systemc = conf.env +with gem5_scons.Configure(systemc) as conf: + if systemc['PLATFORM'] == 'darwin': + systemc.Append(LINKFLAGS=['-undefined', 'dynamic_lookup']) -if systemc['PLATFORM'] == 'darwin': - systemc.Append(LINKFLAGS=['-undefined', 'dynamic_lookup']) - -arch = None -systemc['COROUTINE_LIB'] = '' -if conf.CheckDeclaration('__i386__'): - systemc['COROUTINE_LIB'] = 'qt' - systemc['QT_ARCH'] = 'i386' - arch = 'i386' -elif conf.CheckDeclaration('__x86_64__'): - systemc['COROUTINE_LIB'] = 'qt' - systemc['QT_ARCH'] = 'iX86_64' - arch = 'x86_64' -else: - termcap = get_termcap(GetOption('use_colors')) - print(termcap.Yellow + termcap.Bold + - "Warning: Unrecognized architecture for systemc." + termcap.Normal) - -systemc = conf.Finish() + arch = None + systemc['COROUTINE_LIB'] = '' + if conf.CheckDeclaration('__i386__'): + systemc['COROUTINE_LIB'] = 'qt' + systemc['QT_ARCH'] = 'i386' + arch = 'i386' + elif conf.CheckDeclaration('__x86_64__'): + systemc['COROUTINE_LIB'] = 'qt' + systemc['QT_ARCH'] = 'iX86_64' + arch = 'x86_64' + else: + termcap = get_termcap(GetOption('use_colors')) + print(termcap.Yellow + termcap.Bold + + "Warning: Unrecognized architecture for systemc." + termcap.Normal) if systemc['COROUTINE_LIB'] == 'pthreads': systemc.Prepend(CXXFLAGS=['-DSC_USE_PTHREADS']) @@ -77,7 +72,8 @@ build_dir = os.path.relpath(root, src_root) systemc.SConscript(os.path.join(root, 'SConscript.sc'), exports=['systemc', 'SystemCSource'], - variant_dir=os.path.join(build_root, build_dir)) + variant_dir=os.path.join(build_root, build_dir), + duplicate=GetOption('duplicate_sources')) systemc.Library('libsystemc', systemc_files) systemc.SharedLibrary('libsystemc', systemc_files) diff --git a/site_scons/gem5_scons/sources.py b/site_scons/gem5_scons/sources.py index 548e938..54aeb24 100644 --- a/site_scons/gem5_scons/sources.py +++ b/site_scons/gem5_scons/sources.py @@ -261,13 +261,13 @@ if self.append: env = env.Clone() env.Append(**self.append) - return env.StaticObject(self.tnode) + return env.StaticObject(self.tnode.abspath) def shared(self, env): if self.append: env = env.Clone() env.Append(**self.append) - return env.SharedObject(self.tnode) + return env.SharedObject(self.tnode.abspath) __all__ = [ diff --git a/src/SConscript b/src/SConscript index 4e9048c..13f08d2 100644 --- a/src/SConscript +++ b/src/SConscript @@ -86,10 +86,10 @@ # as gem5. This is in an unorthodox location to avoid building it for every # variant. gem5py_env = gem5py_env.Clone() -gem5py = gem5py_env.File('gem5py') -gem5py_m5 = gem5py_env.File('gem5py_m5') -gem5py_env['GEM5PY'] = gem5py -gem5py_env['GEM5PY_M5'] = gem5py_m5 +gem5py = gem5py_env.File('gem5py', Dir(gem5py_env['BUILDDIR'])) +gem5py_m5 = gem5py_env.File('gem5py_m5', Dir(gem5py_env['BUILDDIR'])) +gem5py_env['GEM5PY'] = gem5py.get_abspath() +gem5py_env['GEM5PY_M5'] = gem5py_m5.get_abspath() gem5py_env['OBJSUFFIX'] = '.pyo' # Inject build_tools into PYTHONPATH for when we run gem5py. pythonpath = gem5py_env['ENV'].get('PYTHONPATH', '').split(':') @@ -121,7 +121,7 @@ self.modpath = modpath - cpp = File(self.filename + '.cc') + cpp = self.tnode.target_from_source('', '.py.cc').get_abspath() overrides = { 'PYSOURCE_MODPATH': modpath, @@ -169,12 +169,13 @@ return ' '.join(list('"${%s}"' % arg for arg in all_args)) # Params header. + params_hh = build_dir.File(f'params/{simobj}.hh').get_abspath() gem5py_env.Command([ "${PARAMS_HH}" ], srcs, MakeAction(cmdline('PARAMS_HH'), Transform("SO Param", 2)), MODULE=module, SIMOBJ=simobj, PYSCRIPT=build_tools.File('sim_object_param_struct_hh.py'), - PARAMS_HH=build_dir.File(f'params/{simobj}.hh')) + PARAMS_HH=params_hh) # Params cc. cc_file = build_dir.File(f'python/_m5/param_{simobj}.cc') @@ -184,18 +185,19 @@ PYSCRIPT=build_tools.File('sim_object_param_struct_cc.py'), MODULE=module, SIMOBJ=simobj, - PARAMS_CC=cc_file, + PARAMS_CC=cc_file.get_abspath(), USE_PYTHON=env['USE_PYTHON']) - Source(cc_file, tags=self.tags, + Source(cc_file.get_abspath(), tags=self.tags, add_tags=('python' if env['USE_PYTHON'] else None)) # CXX config header. + config_hh = build_dir.File(f'cxx_config/{simobj}.hh').get_abspath() gem5py_env.Command([ "${CXXCONFIG_HH}" ], srcs, MakeAction(cmdline('CXXCONFIG_HH'), Transform("CXXCPRHH", 2)), PYSCRIPT=build_tools.File('cxx_config_hh.py'), MODULE=module, - CXXCONFIG_HH=build_dir.File(f'cxx_config/{simobj}.hh')) + CXXCONFIG_HH=config_hh) # CXX config cc. cc_file=build_dir.File(f'cxx_config/{simobj}.cc') @@ -204,9 +206,9 @@ Transform("CXXCPRCC", 2)), PYSCRIPT=build_tools.File('cxx_config_cc.py'), MODULE=module, - CXXCONFIG_CC=cc_file) + CXXCONFIG_CC=cc_file.get_abspath()) if GetOption('with_cxx_config'): - Source(cc_file, tags=self.tags) + Source(cc_file.get_abspath(), tags=self.tags) # C++ versions of enum params. for enum in enums: @@ -218,7 +220,7 @@ Transform("ENUMDECL", 2)), MODULE=module, ENUM=enum, - ENUM_HH=build_dir.File(f'enums/{enum}.hh'), + ENUM_HH=build_dir.File(f'enums/{enum}.hh').get_abspath(), ENUMHH_PY=build_tools.File('enum_hh.py')) cc_file = build_dir.File(f'enums/{enum}.cc') gem5py_env.Command([ "${ENUM_CC}" ], @@ -229,10 +231,10 @@ Transform("ENUM STR", 2)), MODULE=module, ENUM=enum, - ENUM_CC=cc_file, + ENUM_CC=cc_file.get_abspath(), ENUMCC_PY=build_tools.File('enum_cc.py'), USE_PYTHON=env['USE_PYTHON']) - Source(cc_file, tags=self.tags, + Source(cc_file.get_abspath(), tags=self.tags, add_tags=('python' if env['USE_PYTHON'] else None)) # This regular expression is simplistic and assumes that the import takes up @@ -415,8 +417,9 @@ cmd = 'cp $SOURCE $TARGET; strip $TARGET' else: cmd = 'strip $SOURCE -o $TARGET' - stripped = env.Command(str(executable) + '.stripped', - executable, MakeAction(cmd, Transform("STRIP")))[0] + stripped = env.Command(executable.abspath + '.stripped', + executable.abspath, + MakeAction(cmd, Transform("STRIP")))[0] return [executable, stripped] @@ -550,7 +553,8 @@ if 'SConscript' in files: build_dir = os.path.join(env['BUILDDIR'], root[len(base_dir) + 1:]) - SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir) + SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir, + duplicate=GetOption('duplicate_sources')) for extra_dir in extras_dir_list: prefix_len = len(os.path.dirname(extra_dir)) + 1 @@ -566,7 +570,8 @@ if 'SConscript' in files: build_dir = os.path.join(env['BUILDDIR'], root[prefix_len:]) - SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir) + SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir, + duplicate=GetOption('duplicate_sources')) for opt in env['CONF'].keys(): env.ConfigFile(opt) diff --git a/src/mem/slicc/symbols/SymbolTable.py b/src/mem/slicc/symbols/SymbolTable.py index f5dfec1..d2fbf8f 100644 --- a/src/mem/slicc/symbols/SymbolTable.py +++ b/src/mem/slicc/symbols/SymbolTable.py @@ -40,7 +40,7 @@ if not os.path.isdir(path): raise AttributeError(f"{path} exists but is not directory") else: - os.mkdir(path) + os.makedirs(path, exist_ok=True) class SymbolTable(object): diff --git a/util/m5/SConstruct b/util/m5/SConstruct index 62be63c..c2c4a50 100644 --- a/util/m5/SConstruct +++ b/util/m5/SConstruct @@ -179,10 +179,12 @@ # Bring in the googletest sources. native.SConscript(googletest_dir.File('SConscript'), - variant_dir=native_dir.Dir('googletest'), exports={ 'env': native }) + variant_dir=native_dir.Dir('googletest'), exports={ 'env': native }, + duplicate=GetOption('duplicate_sources')) native.SConscript(src_dir.File('SConscript.native'), - variant_dir=native_dir, exports={ 'env': native }) + variant_dir=native_dir, exports={ 'env': native }, + duplicate=GetOption('duplicate_sources')) main['CC'] = '${CROSS_COMPILE}gcc' main['CXX'] = '${CROSS_COMPILE}g++' @@ -268,6 +270,7 @@ # Bring in the googletest sources. env.SConscript(googletest_dir.File('SConscript'), variant_dir=abi_dir.Dir('googletest'), - exports='env') + exports='env', duplicate=GetOption('duplicate_sources')) env.SConscript(src_dir.File('SConscript'), - variant_dir=abi_dir, exports='env') + variant_dir=abi_dir, exports='env', + duplicate=GetOption('duplicate_sources')) diff --git a/util/statetrace/SConstruct b/util/statetrace/SConstruct index 282c154..945976e 100644 --- a/util/statetrace/SConstruct +++ b/util/statetrace/SConstruct @@ -62,4 +62,5 @@ env['CXX'] = ARGUMENTS.get(arch.upper() + 'CXX', env['CXX']) env.Append(CPPFLAGS = '-D__STATETRACE_%s__' % arch.upper()) Export('env', 'arch') - env.SConscript('SConscript', variant_dir = os.path.join('build', arch)) + env.SConscript('SConscript', variant_dir=os.path.join('build', arch), + duplicate=GetOption('duplicate_sources')) diff --git a/util/tlm/SConstruct b/util/tlm/SConstruct index c05b70b..6c65cfd 100644 --- a/util/tlm/SConstruct +++ b/util/tlm/SConstruct @@ -66,11 +66,13 @@ deps = [] # keep track of all dependencies required for building the binaries -deps += SConscript('src/SConscript', variant_dir='build/tlm', exports='env') +deps += SConscript('src/SConscript', variant_dir='build/tlm', exports='env', + duplicate=GetOption('duplicate_sources')) deps += SConscript('examples/common/SConscript', variant_dir='build/examples/common', - exports=['env']) + exports=['env'], + duplicate=GetOption('duplicate_sources')) # the SystemC SConscript makes certain assumptions, we need to fulfill these # assumptions before calling the SConscript. @@ -81,7 +83,7 @@ env.SConsignFile('build/systemc/sconsign') SConscript(gem5_root + '/ext/systemc/SConscript', variant_dir='build/systemc', - exports='env') + exports='env', duplicate=GetOption('duplicate_sources')) # By adding libraries as dependencies instead of using LIBS, we avoid that # the user needs to set the LD_LIBRARY_PATH @@ -91,10 +93,12 @@ ex_master = SConscript('examples/master_port/SConscript', variant_dir='build/examples/master_port', - exports=['env', 'deps']) + exports=['env', 'deps'], + duplicate=GetOption('duplicate_sources')) ex_slave = SConscript('examples/slave_port/SConscript', variant_dir='build/examples/slave_port', - exports=['env', 'deps']) + exports=['env', 'deps'], + duplicate=GetOption('duplicate_sources')) Default(ex_master + ex_slave) -- To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/68518?usp=email To unsubscribe, or for help writing mail filters, visit https://gem5-review.googlesource.com/settings Gerrit-Project: public/gem5 Gerrit-Branch: develop Gerrit-Change-Id: Iddc9bf9c8211e68e5432c0a07f5c95f427c1ca16 Gerrit-Change-Number: 68518 Gerrit-PatchSet: 8 Gerrit-Owner: Alex Richardson <alexrichardson@google.com> Gerrit-Reviewer: Alex Richardson <alexrichardson@google.com> Gerrit-Reviewer: Bobby Bruce <bbruce@ucdavis.edu> Gerrit-Reviewer: Gabe Black <gabe.black@gmail.com> Gerrit-Reviewer: Gabe Black <gabeblack@google.com> Gerrit-Reviewer: kokoro <noreply+kokoro@google.com> Gerrit-MessageType: merged