From 5e096f5b5d1b6a89ec6343092189b186a69d8f48 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Thu, 23 Feb 2023 11:13:26 +0000 Subject: [PATCH] scons: allow building without duplicating source files This adds a new scons flag --no-duplicate-sources to build without linking source files to the build directory. I find this very helpful when using CLion, since I can now generate a compilation database using `bear scons build/ALL/gem5.debug --no-duplicate-sources` and CLion will now correctly semantically analyze all the files inside src/. It also ensures that clicking on a build warning/error now opens the real source file rather than a symlink. This is not enabled by default since it's possible that certain use cases are not working correctly, but the basic testing I've done so far appears to work just fine. It appears that with this change the `/src` directory is no longer added to `PYTHONPATH` when running `tests/main.py`, so this change depends on https://gem5-review.git.corp.google.com/c/public/gem5/+/68757 Change-Id: Iddc9bf9c8211e68e5432c0a07f5c95f427c1ca16 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/68518 Tested-by: kokoro Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce --- SConstruct | 10 ++++-- ext/drampower/SConscript | 2 +- ext/dramsim2/SConscript | 2 +- ext/dramsim3/SConscript | 4 +-- ext/fputils/SConscript | 2 +- ext/iostream3/SConscript | 2 +- ext/libelf/SConscript | 15 +++++---- ext/libfdt/SConscript | 2 +- ext/nomali/SConscript | 2 +- ext/softfloat/SConscript | 2 +- ext/systemc/SConscript | 46 +++++++++++++--------------- site_scons/gem5_scons/sources.py | 4 +-- src/SConscript | 41 ++++++++++++++----------- src/mem/slicc/symbols/SymbolTable.py | 2 +- util/m5/SConstruct | 11 ++++--- util/statetrace/SConstruct | 3 +- util/tlm/SConstruct | 14 ++++++--- 17 files changed, 91 insertions(+), 73 deletions(-) diff --git a/SConstruct b/SConstruct index 6abbb51e00..7d6f40624d 100755 --- a/SConstruct +++ b/SConstruct @@ -145,6 +145,9 @@ AddOption('--gprof', action='store_true', help='Enable support for the gprof profiler') AddOption('--pprof', action='store_true', help='Enable support for the pprof profiler') +AddOption('--no-duplicate-sources', action='store_false', + dest='duplicate_sources', + help='Do not create symlinks to sources in the build directory') # Inject the built_tools directory into the python path. sys.path[1:1] = [ Dir('#build_tools').abspath ] @@ -264,6 +267,7 @@ main.Append(CPPPATH=[Dir('ext')]) # Add shared top-level headers main.Prepend(CPPPATH=Dir('include')) +main.Prepend(CPPPATH=Dir('src')) ######################################################################## @@ -774,11 +778,13 @@ Build variables for {dir}: build_dir = os.path.relpath(root, ext_dir) SConscript(os.path.join(root, 'SConscript'), variant_dir=os.path.join(variant_ext, build_dir), - exports=exports) + exports=exports, + duplicate=GetOption('duplicate_sources')) # The src/SConscript file sets up the build rules in 'env' according # to the configured variables. It returns a list of environments, # one for each variant build (debug, opt, etc.) - SConscript('src/SConscript', variant_dir=variant_path, exports=exports) + SConscript('src/SConscript', variant_dir=variant_path, exports=exports, + duplicate=GetOption('duplicate_sources')) atexit.register(summarize_warnings) diff --git a/ext/drampower/SConscript b/ext/drampower/SConscript index 870d0504c7..38acbf4d06 100644 --- a/ext/drampower/SConscript +++ b/ext/drampower/SConscript @@ -41,7 +41,7 @@ import os Import('env') -env.Prepend(CPPPATH=Dir('./src')) +env.Prepend(CPPPATH=Dir('./src').srcnode()) # Add the appropriate files for the library drampower_files = [] diff --git a/ext/dramsim2/SConscript b/ext/dramsim2/SConscript index 7eb178d626..c2965384d5 100644 --- a/ext/dramsim2/SConscript +++ b/ext/dramsim2/SConscript @@ -85,6 +85,6 @@ dramenv.Append(CCFLAGS=['-DNO_STORAGE']) dramenv.Library('dramsim2', [dramenv.SharedObject(f) for f in dram_files]) -env.Prepend(CPPPATH=Dir('.')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['dramsim2']) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/dramsim3/SConscript b/ext/dramsim3/SConscript index b7178161f7..6be9690db2 100644 --- a/ext/dramsim3/SConscript +++ b/ext/dramsim3/SConscript @@ -56,12 +56,12 @@ dramsim_path = os.path.join(Dir('#').abspath, 'ext/dramsim3/DRAMsim3/') if thermal: superlu_path = os.path.join(dramsim_path, 'ext/SuperLU_MT_3.1/lib') - env.Prepend(CPPPATH=Dir('.')) + env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['dramsim3', 'superlu_mt_OPENMP', 'm', 'f77blas', 'atlas', 'gomp'], LIBPATH=[dramsim_path, superlu_path]) else: - env.Prepend(CPPPATH=Dir('.')) + env.Prepend(CPPPATH=Dir('.').srcnode()) # a littel hacky but can get a shared library working env.Append(LIBS=['dramsim3', 'gomp'], LIBPATH=[dramsim_path], # compile-time lookup diff --git a/ext/fputils/SConscript b/ext/fputils/SConscript index 6a8e44f4af..bc158c2936 100644 --- a/ext/fputils/SConscript +++ b/ext/fputils/SConscript @@ -30,7 +30,7 @@ Import('env') -env.Prepend(CPPPATH=Dir('./include')) +env.Prepend(CPPPATH=Dir('./include').srcnode()) fpenv = env.Clone() diff --git a/ext/iostream3/SConscript b/ext/iostream3/SConscript index df0b2132f2..3b4e93701d 100644 --- a/ext/iostream3/SConscript +++ b/ext/iostream3/SConscript @@ -41,6 +41,6 @@ Import('env') env.Library('iostream3', [env.SharedObject('zfstream.cc')]) -env.Prepend(CPPPATH=Dir('.')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['iostream3']) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/libelf/SConscript b/ext/libelf/SConscript index 535e216ddf..d6f8234bda 100644 --- a/ext/libelf/SConscript +++ b/ext/libelf/SConscript @@ -127,16 +127,19 @@ if not SCons.Tool.m4.exists(m4env): # Setup m4 tool m4env.Tool('m4') -m4env.Append(M4FLAGS=['-DSRCDIR=%s' % Dir('.').path]) +m4env.Append(M4FLAGS=['-DSRCDIR=%s' % Dir('.').srcnode().path]) m4env['M4COM'] = '$M4 $M4FLAGS $SOURCES > $TARGET' m4env.M4(target=File('libelf_convert.c'), - source=[File('elf_types.m4'), File('libelf_convert.m4')]) + source=[File('elf_types.m4').srcnode(), + File('libelf_convert.m4').srcnode()]) m4env.M4(target=File('libelf_fsize.c'), - source=[File('elf_types.m4'), File('libelf_fsize.m4')]) + source=[File('elf_types.m4').srcnode(), + File('libelf_fsize.m4').srcnode()]) m4env.M4(target=File('libelf_msize.c'), - source=[File('elf_types.m4'), File('libelf_msize.m4')]) + source=[File('elf_types.m4').srcnode(), + File('libelf_msize.m4').srcnode()]) -m4env.Append(CPPPATH=Dir('.')) +m4env.Append(CPPPATH=[Dir('.'), Dir('.').srcnode()]) # Build libelf as a static library with PIC code so it can be linked # into either m5 or the library @@ -146,6 +149,6 @@ m4env.Library('elf', [m4env.SharedObject(f) for f in elf_files]) m4env.Command(File('native-elf-format.h'), File('native-elf-format'), '${SOURCE} > ${TARGET}') -env.Prepend(CPPPATH=Dir('.')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=[File('libelf.a')]) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/libfdt/SConscript b/ext/libfdt/SConscript index 64573b78c8..a509bbebad 100644 --- a/ext/libfdt/SConscript +++ b/ext/libfdt/SConscript @@ -44,6 +44,6 @@ FdtFile('fdt_empty_tree.c') FdtFile('fdt_strerror.c') env.Library('fdt', [env.SharedObject(f) for f in fdt_files]) -env.Prepend(CPPPATH=Dir('.')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['fdt']) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/nomali/SConscript b/ext/nomali/SConscript index b156ab0b3e..bcc5cfbd7b 100644 --- a/ext/nomali/SConscript +++ b/ext/nomali/SConscript @@ -39,7 +39,7 @@ Import('env') -env.Prepend(CPPPATH=Dir('./include')) +env.Prepend(CPPPATH=Dir('include').srcnode()) nomali = env.Clone() nomali.Append(CCFLAGS=['-Wno-ignored-qualifiers']) diff --git a/ext/softfloat/SConscript b/ext/softfloat/SConscript index b4a8d514f5..420a71e3f9 100644 --- a/ext/softfloat/SConscript +++ b/ext/softfloat/SConscript @@ -420,6 +420,6 @@ else: sf_env.Library('softfloat', [sf_env.SharedObject(f) for f in softfloat_files]) -env.Prepend(CPPPATH=Dir('./')) +env.Prepend(CPPPATH=Dir('.').srcnode()) env.Append(LIBS=['softfloat']) env.Prepend(LIBPATH=[Dir('.')]) diff --git a/ext/systemc/SConscript b/ext/systemc/SConscript index 89ef9202bd..5248fc32d9 100644 --- a/ext/systemc/SConscript +++ b/ext/systemc/SConscript @@ -25,6 +25,7 @@ import os from m5.util.terminal import get_termcap +import gem5_scons Import('env') systemc = env.Clone() @@ -32,36 +33,30 @@ systemc = env.Clone() build_root = Dir('.').abspath src_root = Dir('.').srcdir.abspath -systemc.Prepend(CPPPATH=Dir('./src')) +systemc.Prepend(CPPPATH=Dir('./src').srcnode()) systemc.Prepend(CPATH=Dir('./src')) systemc.Prepend(CXXFLAGS=['-DSC_INCLUDE_FX']) systemc.Prepend(CFLAGS=['-DSC_INCLUDE_FX']) -conf = Configure(systemc, - conf_dir = os.path.join(build_root, '.scons_config'), - log_file = os.path.join(build_root, 'scons_config.log')) -systemc = conf.env +with gem5_scons.Configure(systemc) as conf: + if systemc['PLATFORM'] == 'darwin': + systemc.Append(LINKFLAGS=['-undefined', 'dynamic_lookup']) -if systemc['PLATFORM'] == 'darwin': - systemc.Append(LINKFLAGS=['-undefined', 'dynamic_lookup']) - -arch = None -systemc['COROUTINE_LIB'] = '' -if conf.CheckDeclaration('__i386__'): - systemc['COROUTINE_LIB'] = 'qt' - systemc['QT_ARCH'] = 'i386' - arch = 'i386' -elif conf.CheckDeclaration('__x86_64__'): - systemc['COROUTINE_LIB'] = 'qt' - systemc['QT_ARCH'] = 'iX86_64' - arch = 'x86_64' -else: - termcap = get_termcap(GetOption('use_colors')) - print(termcap.Yellow + termcap.Bold + - "Warning: Unrecognized architecture for systemc." + termcap.Normal) - -systemc = conf.Finish() + arch = None + systemc['COROUTINE_LIB'] = '' + if conf.CheckDeclaration('__i386__'): + systemc['COROUTINE_LIB'] = 'qt' + systemc['QT_ARCH'] = 'i386' + arch = 'i386' + elif conf.CheckDeclaration('__x86_64__'): + systemc['COROUTINE_LIB'] = 'qt' + systemc['QT_ARCH'] = 'iX86_64' + arch = 'x86_64' + else: + termcap = get_termcap(GetOption('use_colors')) + print(termcap.Yellow + termcap.Bold + + "Warning: Unrecognized architecture for systemc." + termcap.Normal) if systemc['COROUTINE_LIB'] == 'pthreads': systemc.Prepend(CXXFLAGS=['-DSC_USE_PTHREADS']) @@ -77,7 +72,8 @@ if arch: build_dir = os.path.relpath(root, src_root) systemc.SConscript(os.path.join(root, 'SConscript.sc'), exports=['systemc', 'SystemCSource'], - variant_dir=os.path.join(build_root, build_dir)) + variant_dir=os.path.join(build_root, build_dir), + duplicate=GetOption('duplicate_sources')) systemc.Library('libsystemc', systemc_files) systemc.SharedLibrary('libsystemc', systemc_files) diff --git a/site_scons/gem5_scons/sources.py b/site_scons/gem5_scons/sources.py index 548e9386ea..54aeb24de1 100644 --- a/site_scons/gem5_scons/sources.py +++ b/site_scons/gem5_scons/sources.py @@ -261,13 +261,13 @@ class SourceFile(SourceItem): if self.append: env = env.Clone() env.Append(**self.append) - return env.StaticObject(self.tnode) + return env.StaticObject(self.tnode.abspath) def shared(self, env): if self.append: env = env.Clone() env.Append(**self.append) - return env.SharedObject(self.tnode) + return env.SharedObject(self.tnode.abspath) __all__ = [ diff --git a/src/SConscript b/src/SConscript index 4e9048ca04..13f08d2f5a 100644 --- a/src/SConscript +++ b/src/SConscript @@ -86,10 +86,10 @@ build_tools = Dir('#build_tools') # as gem5. This is in an unorthodox location to avoid building it for every # variant. gem5py_env = gem5py_env.Clone() -gem5py = gem5py_env.File('gem5py') -gem5py_m5 = gem5py_env.File('gem5py_m5') -gem5py_env['GEM5PY'] = gem5py -gem5py_env['GEM5PY_M5'] = gem5py_m5 +gem5py = gem5py_env.File('gem5py', Dir(gem5py_env['BUILDDIR'])) +gem5py_m5 = gem5py_env.File('gem5py_m5', Dir(gem5py_env['BUILDDIR'])) +gem5py_env['GEM5PY'] = gem5py.get_abspath() +gem5py_env['GEM5PY_M5'] = gem5py_m5.get_abspath() gem5py_env['OBJSUFFIX'] = '.pyo' # Inject build_tools into PYTHONPATH for when we run gem5py. pythonpath = gem5py_env['ENV'].get('PYTHONPATH', '').split(':') @@ -121,7 +121,7 @@ class PySource(SourceFile): self.modpath = modpath - cpp = File(self.filename + '.cc') + cpp = self.tnode.target_from_source('', '.py.cc').get_abspath() overrides = { 'PYSOURCE_MODPATH': modpath, @@ -169,12 +169,13 @@ class SimObject(PySource): return ' '.join(list('"${%s}"' % arg for arg in all_args)) # Params header. + params_hh = build_dir.File(f'params/{simobj}.hh').get_abspath() gem5py_env.Command([ "${PARAMS_HH}" ], srcs, MakeAction(cmdline('PARAMS_HH'), Transform("SO Param", 2)), MODULE=module, SIMOBJ=simobj, PYSCRIPT=build_tools.File('sim_object_param_struct_hh.py'), - PARAMS_HH=build_dir.File(f'params/{simobj}.hh')) + PARAMS_HH=params_hh) # Params cc. cc_file = build_dir.File(f'python/_m5/param_{simobj}.cc') @@ -184,18 +185,19 @@ class SimObject(PySource): PYSCRIPT=build_tools.File('sim_object_param_struct_cc.py'), MODULE=module, SIMOBJ=simobj, - PARAMS_CC=cc_file, + PARAMS_CC=cc_file.get_abspath(), USE_PYTHON=env['USE_PYTHON']) - Source(cc_file, tags=self.tags, + Source(cc_file.get_abspath(), tags=self.tags, add_tags=('python' if env['USE_PYTHON'] else None)) # CXX config header. + config_hh = build_dir.File(f'cxx_config/{simobj}.hh').get_abspath() gem5py_env.Command([ "${CXXCONFIG_HH}" ], srcs, MakeAction(cmdline('CXXCONFIG_HH'), Transform("CXXCPRHH", 2)), PYSCRIPT=build_tools.File('cxx_config_hh.py'), MODULE=module, - CXXCONFIG_HH=build_dir.File(f'cxx_config/{simobj}.hh')) + CXXCONFIG_HH=config_hh) # CXX config cc. cc_file=build_dir.File(f'cxx_config/{simobj}.cc') @@ -204,9 +206,9 @@ class SimObject(PySource): Transform("CXXCPRCC", 2)), PYSCRIPT=build_tools.File('cxx_config_cc.py'), MODULE=module, - CXXCONFIG_CC=cc_file) + CXXCONFIG_CC=cc_file.get_abspath()) if GetOption('with_cxx_config'): - Source(cc_file, tags=self.tags) + Source(cc_file.get_abspath(), tags=self.tags) # C++ versions of enum params. for enum in enums: @@ -218,7 +220,7 @@ class SimObject(PySource): Transform("ENUMDECL", 2)), MODULE=module, ENUM=enum, - ENUM_HH=build_dir.File(f'enums/{enum}.hh'), + ENUM_HH=build_dir.File(f'enums/{enum}.hh').get_abspath(), ENUMHH_PY=build_tools.File('enum_hh.py')) cc_file = build_dir.File(f'enums/{enum}.cc') gem5py_env.Command([ "${ENUM_CC}" ], @@ -229,10 +231,10 @@ class SimObject(PySource): Transform("ENUM STR", 2)), MODULE=module, ENUM=enum, - ENUM_CC=cc_file, + ENUM_CC=cc_file.get_abspath(), ENUMCC_PY=build_tools.File('enum_cc.py'), USE_PYTHON=env['USE_PYTHON']) - Source(cc_file, tags=self.tags, + Source(cc_file.get_abspath(), tags=self.tags, add_tags=('python' if env['USE_PYTHON'] else None)) # This regular expression is simplistic and assumes that the import takes up @@ -415,8 +417,9 @@ class Executable(TopLevelBase): cmd = 'cp $SOURCE $TARGET; strip $TARGET' else: cmd = 'strip $SOURCE -o $TARGET' - stripped = env.Command(str(executable) + '.stripped', - executable, MakeAction(cmd, Transform("STRIP")))[0] + stripped = env.Command(executable.abspath + '.stripped', + executable.abspath, + MakeAction(cmd, Transform("STRIP")))[0] return [executable, stripped] @@ -550,7 +553,8 @@ for root, dirs, files in os.walk(base_dir, topdown=True): if 'SConscript' in files: build_dir = os.path.join(env['BUILDDIR'], root[len(base_dir) + 1:]) - SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir) + SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir, + duplicate=GetOption('duplicate_sources')) for extra_dir in extras_dir_list: prefix_len = len(os.path.dirname(extra_dir)) + 1 @@ -566,7 +570,8 @@ for extra_dir in extras_dir_list: if 'SConscript' in files: build_dir = os.path.join(env['BUILDDIR'], root[prefix_len:]) - SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir) + SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir, + duplicate=GetOption('duplicate_sources')) for opt in env['CONF'].keys(): env.ConfigFile(opt) diff --git a/src/mem/slicc/symbols/SymbolTable.py b/src/mem/slicc/symbols/SymbolTable.py index f5dfec1d68..d2fbf8f7a9 100644 --- a/src/mem/slicc/symbols/SymbolTable.py +++ b/src/mem/slicc/symbols/SymbolTable.py @@ -40,7 +40,7 @@ def makeDir(path): if not os.path.isdir(path): raise AttributeError(f"{path} exists but is not directory") else: - os.mkdir(path) + os.makedirs(path, exist_ok=True) class SymbolTable(object): diff --git a/util/m5/SConstruct b/util/m5/SConstruct index 62be63c66a..c2c4a50a95 100644 --- a/util/m5/SConstruct +++ b/util/m5/SConstruct @@ -179,10 +179,12 @@ native_dir = build_dir.Dir('native') # Bring in the googletest sources. native.SConscript(googletest_dir.File('SConscript'), - variant_dir=native_dir.Dir('googletest'), exports={ 'env': native }) + variant_dir=native_dir.Dir('googletest'), exports={ 'env': native }, + duplicate=GetOption('duplicate_sources')) native.SConscript(src_dir.File('SConscript.native'), - variant_dir=native_dir, exports={ 'env': native }) + variant_dir=native_dir, exports={ 'env': native }, + duplicate=GetOption('duplicate_sources')) main['CC'] = '${CROSS_COMPILE}gcc' main['CXX'] = '${CROSS_COMPILE}g++' @@ -268,6 +270,7 @@ for root, dirs, files in os.walk(abspath(src_dir)): # Bring in the googletest sources. env.SConscript(googletest_dir.File('SConscript'), variant_dir=abi_dir.Dir('googletest'), - exports='env') + exports='env', duplicate=GetOption('duplicate_sources')) env.SConscript(src_dir.File('SConscript'), - variant_dir=abi_dir, exports='env') + variant_dir=abi_dir, exports='env', + duplicate=GetOption('duplicate_sources')) diff --git a/util/statetrace/SConstruct b/util/statetrace/SConstruct index 282c1543a8..945976e8c5 100644 --- a/util/statetrace/SConstruct +++ b/util/statetrace/SConstruct @@ -62,4 +62,5 @@ for arch in arches: env['CXX'] = ARGUMENTS.get(arch.upper() + 'CXX', env['CXX']) env.Append(CPPFLAGS = '-D__STATETRACE_%s__' % arch.upper()) Export('env', 'arch') - env.SConscript('SConscript', variant_dir = os.path.join('build', arch)) + env.SConscript('SConscript', variant_dir=os.path.join('build', arch), + duplicate=GetOption('duplicate_sources')) diff --git a/util/tlm/SConstruct b/util/tlm/SConstruct index c05b70b208..6c65cfddfa 100644 --- a/util/tlm/SConstruct +++ b/util/tlm/SConstruct @@ -66,11 +66,13 @@ if gem5_variant == 'debug': deps = [] # keep track of all dependencies required for building the binaries -deps += SConscript('src/SConscript', variant_dir='build/tlm', exports='env') +deps += SConscript('src/SConscript', variant_dir='build/tlm', exports='env', + duplicate=GetOption('duplicate_sources')) deps += SConscript('examples/common/SConscript', variant_dir='build/examples/common', - exports=['env']) + exports=['env'], + duplicate=GetOption('duplicate_sources')) # the SystemC SConscript makes certain assumptions, we need to fulfill these # assumptions before calling the SConscript. @@ -81,7 +83,7 @@ AddOption('--no-colors', dest='use_colors', action='store_false', env.SConsignFile('build/systemc/sconsign') SConscript(gem5_root + '/ext/systemc/SConscript', variant_dir='build/systemc', - exports='env') + exports='env', duplicate=GetOption('duplicate_sources')) # By adding libraries as dependencies instead of using LIBS, we avoid that # the user needs to set the LD_LIBRARY_PATH @@ -91,10 +93,12 @@ deps.append(File(os.path.join(gem5_root, 'build', gem5_arch, ex_master = SConscript('examples/master_port/SConscript', variant_dir='build/examples/master_port', - exports=['env', 'deps']) + exports=['env', 'deps'], + duplicate=GetOption('duplicate_sources')) ex_slave = SConscript('examples/slave_port/SConscript', variant_dir='build/examples/slave_port', - exports=['env', 'deps']) + exports=['env', 'deps'], + duplicate=GetOption('duplicate_sources')) Default(ex_master + ex_slave)