derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scons: allow building without duplicating source files

Browse Source 
This adds a new scons flag --no-duplicate-sources to build without
linking source files to the build directory.

I find this very helpful when using CLion, since I can now generate a
compilation database using
`bear scons build/ALL/gem5.debug --no-duplicate-sources` and CLion will
now correctly semantically analyze all the files inside src/.
It also ensures that clicking on a build warning/error now opens the
real source file rather than a symlink.

This is not enabled by default since it's possible that certain use
cases are not working correctly, but the basic testing I've done so
far appears to work just fine.

It appears that with this change the `<root>/src` directory is no longer
added to `PYTHONPATH` when running `tests/main.py`, so this change
depends on https://gem5-review.git.corp.google.com/c/public/gem5/+/68757

Change-Id: Iddc9bf9c8211e68e5432c0a07f5c95f427c1ca16
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/68518
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Bobby Bruce <bbruce@ucdavis.edu>
Maintainer: Bobby Bruce <bbruce@ucdavis.edu>
This commit is contained in:
Alex Richardson
2023-02-23 11:13:26 +00:00
parent c00e3b2570
commit 5e096f5b5d
17 changed files with 91 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
SConstruct
View File

@@ -145,6 +145,9 @@ AddOption('--gprof', action='store_true',
help='Enable support for the gprof profiler')
AddOption('--pprof', action='store_true',
help='Enable support for the pprof profiler')
AddOption('--no-duplicate-sources', action='store_false',
dest='duplicate_sources',
help='Do not create symlinks to sources in the build directory')
# Inject the built_tools directory into the python path.
sys.path[1:1] = [ Dir('#build_tools').abspath ]
@@ -264,6 +267,7 @@ main.Append(CPPPATH=[Dir('ext')])
# Add shared top-level headers
main.Prepend(CPPPATH=Dir('include'))
main.Prepend(CPPPATH=Dir('src'))
########################################################################
@@ -774,11 +778,13 @@ Build variables for {dir}:
build_dir = os.path.relpath(root, ext_dir)
SConscript(os.path.join(root, 'SConscript'),
variant_dir=os.path.join(variant_ext, build_dir),
exports=exports)
exports=exports,
duplicate=GetOption('duplicate_sources'))
# The src/SConscript file sets up the build rules in 'env' according
# to the configured variables. It returns a list of environments,
# one for each variant build (debug, opt, etc.)
SConscript('src/SConscript', variant_dir=variant_path, exports=exports)
SConscript('src/SConscript', variant_dir=variant_path, exports=exports,
duplicate=GetOption('duplicate_sources'))
atexit.register(summarize_warnings)

2
ext/drampower/SConscript
View File

@@ -41,7 +41,7 @@ import os
Import('env')
env.Prepend(CPPPATH=Dir('./src'))
env.Prepend(CPPPATH=Dir('./src').srcnode())
# Add the appropriate files for the library
drampower_files = []

2
ext/dramsim2/SConscript
View File

@@ -85,6 +85,6 @@ dramenv.Append(CCFLAGS=['-DNO_STORAGE'])
dramenv.Library('dramsim2', [dramenv.SharedObject(f) for f in dram_files])
env.Prepend(CPPPATH=Dir('.'))
env.Prepend(CPPPATH=Dir('.').srcnode())
env.Append(LIBS=['dramsim2'])
env.Prepend(LIBPATH=[Dir('.')])

4
ext/dramsim3/SConscript
View File

@@ -56,12 +56,12 @@ dramsim_path = os.path.join(Dir('#').abspath, 'ext/dramsim3/DRAMsim3/')
if thermal:
superlu_path = os.path.join(dramsim_path, 'ext/SuperLU_MT_3.1/lib')
env.Prepend(CPPPATH=Dir('.'))
env.Prepend(CPPPATH=Dir('.').srcnode())
env.Append(LIBS=['dramsim3', 'superlu_mt_OPENMP', 'm', 'f77blas',
'atlas', 'gomp'],
LIBPATH=[dramsim_path, superlu_path])
else:
env.Prepend(CPPPATH=Dir('.'))
env.Prepend(CPPPATH=Dir('.').srcnode())
# a littel hacky but can get a shared library working
env.Append(LIBS=['dramsim3', 'gomp'],
LIBPATH=[dramsim_path], # compile-time lookup

2
ext/fputils/SConscript
View File

@@ -30,7 +30,7 @@
Import('env')
env.Prepend(CPPPATH=Dir('./include'))
env.Prepend(CPPPATH=Dir('./include').srcnode())
fpenv = env.Clone()

2
ext/iostream3/SConscript
View File

@@ -41,6 +41,6 @@ Import('env')
env.Library('iostream3', [env.SharedObject('zfstream.cc')])
env.Prepend(CPPPATH=Dir('.'))
env.Prepend(CPPPATH=Dir('.').srcnode())
env.Append(LIBS=['iostream3'])
env.Prepend(LIBPATH=[Dir('.')])

15
ext/libelf/SConscript
View File

@@ -127,16 +127,19 @@ if not SCons.Tool.m4.exists(m4env):
# Setup m4 tool
m4env.Tool('m4')
m4env.Append(M4FLAGS=['-DSRCDIR=%s' % Dir('.').path])
m4env.Append(M4FLAGS=['-DSRCDIR=%s' % Dir('.').srcnode().path])
m4env['M4COM'] = '$M4 $M4FLAGS $SOURCES > $TARGET'
m4env.M4(target=File('libelf_convert.c'),
source=[File('elf_types.m4'), File('libelf_convert.m4')])
source=[File('elf_types.m4').srcnode(),
File('libelf_convert.m4').srcnode()])
m4env.M4(target=File('libelf_fsize.c'),
source=[File('elf_types.m4'), File('libelf_fsize.m4')])
source=[File('elf_types.m4').srcnode(),
File('libelf_fsize.m4').srcnode()])
m4env.M4(target=File('libelf_msize.c'),
source=[File('elf_types.m4'), File('libelf_msize.m4')])
source=[File('elf_types.m4').srcnode(),
File('libelf_msize.m4').srcnode()])
m4env.Append(CPPPATH=Dir('.'))
m4env.Append(CPPPATH=[Dir('.'), Dir('.').srcnode()])
# Build libelf as a static library with PIC code so it can be linked
# into either m5 or the library
@@ -146,6 +149,6 @@ m4env.Library('elf', [m4env.SharedObject(f) for f in elf_files])
m4env.Command(File('native-elf-format.h'), File('native-elf-format'),
'${SOURCE} > ${TARGET}')
env.Prepend(CPPPATH=Dir('.'))
env.Prepend(CPPPATH=Dir('.').srcnode())
env.Append(LIBS=[File('libelf.a')])
env.Prepend(LIBPATH=[Dir('.')])

2
ext/libfdt/SConscript
View File

@@ -44,6 +44,6 @@ FdtFile('fdt_empty_tree.c')
FdtFile('fdt_strerror.c')
env.Library('fdt', [env.SharedObject(f) for f in fdt_files])
env.Prepend(CPPPATH=Dir('.'))
env.Prepend(CPPPATH=Dir('.').srcnode())
env.Append(LIBS=['fdt'])
env.Prepend(LIBPATH=[Dir('.')])

2
ext/nomali/SConscript
View File

@@ -39,7 +39,7 @@
Import('env')
env.Prepend(CPPPATH=Dir('./include'))
env.Prepend(CPPPATH=Dir('include').srcnode())
nomali = env.Clone()
nomali.Append(CCFLAGS=['-Wno-ignored-qualifiers'])

2
ext/softfloat/SConscript
View File

@@ -420,6 +420,6 @@ else:
sf_env.Library('softfloat', [sf_env.SharedObject(f) for f in softfloat_files])
env.Prepend(CPPPATH=Dir('./'))
env.Prepend(CPPPATH=Dir('.').srcnode())
env.Append(LIBS=['softfloat'])
env.Prepend(LIBPATH=[Dir('.')])

46
ext/systemc/SConscript
View File

@@ -25,6 +25,7 @@
import os
from m5.util.terminal import get_termcap
import gem5_scons
Import('env')
systemc = env.Clone()
@@ -32,36 +33,30 @@ systemc = env.Clone()
build_root = Dir('.').abspath
src_root = Dir('.').srcdir.abspath
systemc.Prepend(CPPPATH=Dir('./src'))
systemc.Prepend(CPPPATH=Dir('./src').srcnode())
systemc.Prepend(CPATH=Dir('./src'))
systemc.Prepend(CXXFLAGS=['-DSC_INCLUDE_FX'])
systemc.Prepend(CFLAGS=['-DSC_INCLUDE_FX'])
conf = Configure(systemc,
conf_dir = os.path.join(build_root, '.scons_config'),
log_file = os.path.join(build_root, 'scons_config.log'))
systemc = conf.env
with gem5_scons.Configure(systemc) as conf:
if systemc['PLATFORM'] == 'darwin':
systemc.Append(LINKFLAGS=['-undefined', 'dynamic_lookup'])
if systemc['PLATFORM'] == 'darwin':
systemc.Append(LINKFLAGS=['-undefined', 'dynamic_lookup'])
arch = None
systemc['COROUTINE_LIB'] = ''
if conf.CheckDeclaration('__i386__'):
systemc['COROUTINE_LIB'] = 'qt'
systemc['QT_ARCH'] = 'i386'
arch = 'i386'
elif conf.CheckDeclaration('__x86_64__'):
systemc['COROUTINE_LIB'] = 'qt'
systemc['QT_ARCH'] = 'iX86_64'
arch = 'x86_64'
else:
termcap = get_termcap(GetOption('use_colors'))
print(termcap.Yellow + termcap.Bold +
"Warning: Unrecognized architecture for systemc." + termcap.Normal)
systemc = conf.Finish()
arch = None
systemc['COROUTINE_LIB'] = ''
if conf.CheckDeclaration('__i386__'):
systemc['COROUTINE_LIB'] = 'qt'
systemc['QT_ARCH'] = 'i386'
arch = 'i386'
elif conf.CheckDeclaration('__x86_64__'):
systemc['COROUTINE_LIB'] = 'qt'
systemc['QT_ARCH'] = 'iX86_64'
arch = 'x86_64'
else:
termcap = get_termcap(GetOption('use_colors'))
print(termcap.Yellow + termcap.Bold +
"Warning: Unrecognized architecture for systemc." + termcap.Normal)
if systemc['COROUTINE_LIB'] == 'pthreads':
systemc.Prepend(CXXFLAGS=['-DSC_USE_PTHREADS'])
@@ -77,7 +72,8 @@ if arch:
build_dir = os.path.relpath(root, src_root)
systemc.SConscript(os.path.join(root, 'SConscript.sc'),
exports=['systemc', 'SystemCSource'],
variant_dir=os.path.join(build_root, build_dir))
variant_dir=os.path.join(build_root, build_dir),
duplicate=GetOption('duplicate_sources'))
systemc.Library('libsystemc', systemc_files)
systemc.SharedLibrary('libsystemc', systemc_files)

4
site_scons/gem5_scons/sources.py
View File

@@ -261,13 +261,13 @@ class SourceFile(SourceItem):
if self.append:
env = env.Clone()
env.Append(**self.append)
return env.StaticObject(self.tnode)
return env.StaticObject(self.tnode.abspath)
def shared(self, env):
if self.append:
env = env.Clone()
env.Append(**self.append)
return env.SharedObject(self.tnode)
return env.SharedObject(self.tnode.abspath)
__all__ = [

41
src/SConscript
View File

@@ -86,10 +86,10 @@ build_tools = Dir('#build_tools')
# as gem5. This is in an unorthodox location to avoid building it for every
# variant.
gem5py_env = gem5py_env.Clone()
gem5py = gem5py_env.File('gem5py')
gem5py_m5 = gem5py_env.File('gem5py_m5')
gem5py_env['GEM5PY'] = gem5py
gem5py_env['GEM5PY_M5'] = gem5py_m5
gem5py = gem5py_env.File('gem5py', Dir(gem5py_env['BUILDDIR']))
gem5py_m5 = gem5py_env.File('gem5py_m5', Dir(gem5py_env['BUILDDIR']))
gem5py_env['GEM5PY'] = gem5py.get_abspath()
gem5py_env['GEM5PY_M5'] = gem5py_m5.get_abspath()
gem5py_env['OBJSUFFIX'] = '.pyo'
# Inject build_tools into PYTHONPATH for when we run gem5py.
pythonpath = gem5py_env['ENV'].get('PYTHONPATH', '').split(':')
@@ -121,7 +121,7 @@ class PySource(SourceFile):
self.modpath = modpath
cpp = File(self.filename + '.cc')
cpp = self.tnode.target_from_source('', '.py.cc').get_abspath()
overrides = {
'PYSOURCE_MODPATH': modpath,
@@ -169,12 +169,13 @@ class SimObject(PySource):
return ' '.join(list('"${%s}"' % arg for arg in all_args))
# Params header.
params_hh = build_dir.File(f'params/{simobj}.hh').get_abspath()
gem5py_env.Command([ "${PARAMS_HH}" ], srcs,
MakeAction(cmdline('PARAMS_HH'), Transform("SO Param", 2)),
MODULE=module,
SIMOBJ=simobj,
PYSCRIPT=build_tools.File('sim_object_param_struct_hh.py'),
PARAMS_HH=build_dir.File(f'params/{simobj}.hh'))
PARAMS_HH=params_hh)
# Params cc.
cc_file = build_dir.File(f'python/_m5/param_{simobj}.cc')
@@ -184,18 +185,19 @@ class SimObject(PySource):
PYSCRIPT=build_tools.File('sim_object_param_struct_cc.py'),
MODULE=module,
SIMOBJ=simobj,
PARAMS_CC=cc_file,
PARAMS_CC=cc_file.get_abspath(),
USE_PYTHON=env['USE_PYTHON'])
Source(cc_file, tags=self.tags,
Source(cc_file.get_abspath(), tags=self.tags,
add_tags=('python' if env['USE_PYTHON'] else None))
# CXX config header.
config_hh = build_dir.File(f'cxx_config/{simobj}.hh').get_abspath()
gem5py_env.Command([ "${CXXCONFIG_HH}" ], srcs,
MakeAction(cmdline('CXXCONFIG_HH'),
Transform("CXXCPRHH", 2)),
PYSCRIPT=build_tools.File('cxx_config_hh.py'),
MODULE=module,
CXXCONFIG_HH=build_dir.File(f'cxx_config/{simobj}.hh'))
CXXCONFIG_HH=config_hh)
# CXX config cc.
cc_file=build_dir.File(f'cxx_config/{simobj}.cc')
@@ -204,9 +206,9 @@ class SimObject(PySource):
Transform("CXXCPRCC", 2)),
PYSCRIPT=build_tools.File('cxx_config_cc.py'),
MODULE=module,
CXXCONFIG_CC=cc_file)
CXXCONFIG_CC=cc_file.get_abspath())
if GetOption('with_cxx_config'):
Source(cc_file, tags=self.tags)
Source(cc_file.get_abspath(), tags=self.tags)
# C++ versions of enum params.
for enum in enums:
@@ -218,7 +220,7 @@ class SimObject(PySource):
Transform("ENUMDECL", 2)),
MODULE=module,
ENUM=enum,
ENUM_HH=build_dir.File(f'enums/{enum}.hh'),
ENUM_HH=build_dir.File(f'enums/{enum}.hh').get_abspath(),
ENUMHH_PY=build_tools.File('enum_hh.py'))
cc_file = build_dir.File(f'enums/{enum}.cc')
gem5py_env.Command([ "${ENUM_CC}" ],
@@ -229,10 +231,10 @@ class SimObject(PySource):
Transform("ENUM STR", 2)),
MODULE=module,
ENUM=enum,
ENUM_CC=cc_file,
ENUM_CC=cc_file.get_abspath(),
ENUMCC_PY=build_tools.File('enum_cc.py'),
USE_PYTHON=env['USE_PYTHON'])
Source(cc_file, tags=self.tags,
Source(cc_file.get_abspath(), tags=self.tags,
add_tags=('python' if env['USE_PYTHON'] else None))
# This regular expression is simplistic and assumes that the import takes up
@@ -415,8 +417,9 @@ class Executable(TopLevelBase):
cmd = 'cp $SOURCE $TARGET; strip $TARGET'
else:
cmd = 'strip $SOURCE -o $TARGET'
stripped = env.Command(str(executable) + '.stripped',
executable, MakeAction(cmd, Transform("STRIP")))[0]
stripped = env.Command(executable.abspath + '.stripped',
executable.abspath,
MakeAction(cmd, Transform("STRIP")))[0]
return [executable, stripped]
@@ -550,7 +553,8 @@ for root, dirs, files in os.walk(base_dir, topdown=True):
if 'SConscript' in files:
build_dir = os.path.join(env['BUILDDIR'], root[len(base_dir) + 1:])
SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir)
SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir,
duplicate=GetOption('duplicate_sources'))
for extra_dir in extras_dir_list:
prefix_len = len(os.path.dirname(extra_dir)) + 1
@@ -566,7 +570,8 @@ for extra_dir in extras_dir_list:
if 'SConscript' in files:
build_dir = os.path.join(env['BUILDDIR'], root[prefix_len:])
SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir)
SConscript(os.path.join(root, 'SConscript'), variant_dir=build_dir,
duplicate=GetOption('duplicate_sources'))
for opt in env['CONF'].keys():
env.ConfigFile(opt)

2
src/mem/slicc/symbols/SymbolTable.py
View File

@@ -40,7 +40,7 @@ def makeDir(path):
if not os.path.isdir(path):
raise AttributeError(f"{path} exists but is not directory")
else:
os.mkdir(path)
os.makedirs(path, exist_ok=True)
class SymbolTable(object):

11
util/m5/SConstruct
View File

@@ -179,10 +179,12 @@ native_dir = build_dir.Dir('native')
# Bring in the googletest sources.
native.SConscript(googletest_dir.File('SConscript'),
variant_dir=native_dir.Dir('googletest'), exports={ 'env': native })
variant_dir=native_dir.Dir('googletest'), exports={ 'env': native },
duplicate=GetOption('duplicate_sources'))
native.SConscript(src_dir.File('SConscript.native'),
variant_dir=native_dir, exports={ 'env': native })
variant_dir=native_dir, exports={ 'env': native },
duplicate=GetOption('duplicate_sources'))
main['CC'] = '${CROSS_COMPILE}gcc'
main['CXX'] = '${CROSS_COMPILE}g++'
@@ -268,6 +270,7 @@ for root, dirs, files in os.walk(abspath(src_dir)):
# Bring in the googletest sources.
env.SConscript(googletest_dir.File('SConscript'),
variant_dir=abi_dir.Dir('googletest'),
exports='env')
exports='env', duplicate=GetOption('duplicate_sources'))
env.SConscript(src_dir.File('SConscript'),
variant_dir=abi_dir, exports='env')
variant_dir=abi_dir, exports='env',
duplicate=GetOption('duplicate_sources'))

3
util/statetrace/SConstruct
View File

@@ -62,4 +62,5 @@ for arch in arches:
env['CXX'] = ARGUMENTS.get(arch.upper() + 'CXX', env['CXX'])
env.Append(CPPFLAGS = '-D__STATETRACE_%s__' % arch.upper())
Export('env', 'arch')
env.SConscript('SConscript', variant_dir = os.path.join('build', arch))
env.SConscript('SConscript', variant_dir=os.path.join('build', arch),
duplicate=GetOption('duplicate_sources'))

14
util/tlm/SConstruct
View File

@@ -66,11 +66,13 @@ if gem5_variant == 'debug':
deps = [] # keep track of all dependencies required for building the binaries
deps += SConscript('src/SConscript', variant_dir='build/tlm', exports='env')
deps += SConscript('src/SConscript', variant_dir='build/tlm', exports='env',
duplicate=GetOption('duplicate_sources'))
deps += SConscript('examples/common/SConscript',
variant_dir='build/examples/common',
exports=['env'])
exports=['env'],
duplicate=GetOption('duplicate_sources'))
# the SystemC SConscript makes certain assumptions, we need to fulfill these
# assumptions before calling the SConscript.
@@ -81,7 +83,7 @@ AddOption('--no-colors', dest='use_colors', action='store_false',
env.SConsignFile('build/systemc/sconsign')
SConscript(gem5_root + '/ext/systemc/SConscript',
variant_dir='build/systemc',
exports='env')
exports='env', duplicate=GetOption('duplicate_sources'))
# By adding libraries as dependencies instead of using LIBS, we avoid that
# the user needs to set the LD_LIBRARY_PATH
@@ -91,10 +93,12 @@ deps.append(File(os.path.join(gem5_root, 'build', gem5_arch,
ex_master = SConscript('examples/master_port/SConscript',
variant_dir='build/examples/master_port',
exports=['env', 'deps'])
exports=['env', 'deps'],
duplicate=GetOption('duplicate_sources'))
ex_slave = SConscript('examples/slave_port/SConscript',
variant_dir='build/examples/slave_port',
exports=['env', 'deps'])
exports=['env', 'deps'],
duplicate=GetOption('duplicate_sources'))
Default(ex_master + ex_slave)