gem5/util/cpt_upgrader.py

#!/usr/bin/env python3

# Copyright (c) 2012-2013,2015-2016, 2020 ARM Limited
# All rights reserved
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# This python code is used to migrate checkpoints that were created in one
# version of the simulator to newer version. As features are added or bugs are
# fixed some of the state that needs to be checkpointed can change. If you have
# many historic checkpoints that you use, manually editing them to fix them is
# both time consuming and error-prone.

# This script provides a way to migrate checkpoints to the newer repository in
# a programmatic way. It can be imported into another script or used on the
# command line. From the command line the script will either migrate every
# checkpoint it finds recursively (-r option) or a single checkpoint. When a
# change is made to the gem5 repository that breaks previous checkpoints an
# upgrade() method should be implemented in its own .py file and placed in
# src/util/cpt_upgraders/.  For each upgrader whose tag is not present in
# the checkpoint tag list, the upgrade() method will be run, passing in a
# ConfigParser object which contains the open file. As these operations can
# be isa specific the method can verify the isa and use regexes to find the
# correct sections that need to be updated.

# It is also possible to use this mechanism to revert prior tags.  In this
# case, implement a downgrade() method instead.  Dependencies should still
# work naturally - a tag depending on a tag with a downgrader means that it
# insists on the other tag being removed and its downgrader executed before
# its upgrader (or downgrader) can run.  It is still the case that a tag
# can only be used once.

# Dependencies between tags are expressed by two variables at the top-level
# of the upgrader script: "depends" can be either a string naming another
# tag that it depends upon or a list of such strings; and "fwd_depends"
# accepts the same datatypes but it reverses the sense of the dependency
# arrow(s) -- it expresses that that tag depends upon the tag of the current
# upgrader. This can be especially valuable when maintaining private
# upgraders in private branches.


import configparser
import glob, types, sys, os
import os.path as osp

verbose_print = False

def verboseprint(*args):
    if not verbose_print:
        return
    for arg in args:
        print(arg, end=' ')
    print("\n")

class Upgrader:
    tag_set = set()
    untag_set = set() # tags to remove by downgrading
    by_tag = {}
    legacy = {}
    def __init__(self, filename):
        self.filename = filename
        exec(open(filename).read(), {}, self.__dict__)

        if not hasattr(self, 'tag'):
            self.tag = osp.basename(filename)[:-3]
        if not hasattr(self, 'depends'):
            self.depends = []
        elif isinstance(self.depends, str):
            self.depends = [self.depends]

        if not isinstance(self.depends, list):
            print("Error: 'depends' for {} is the wrong type".format(self.tag))
            sys.exit(1)

        if hasattr(self, 'fwd_depends'):
            if isinstance(self.fwd_depends, str):
                self.fwd_depends = [self.fwd_depends]
        else:
            self.fwd_depends = []

        if not isinstance(self.fwd_depends, list):
            print("Error: 'fwd_depends' for {} is the wrong type".format(
                self.tag))
            sys.exit(1)

        if hasattr(self, 'upgrader'):
            if not isinstance(self.upgrader, types.FunctionType):
                print("Error: 'upgrader' for {} is {}, not function".format(
                    self.tag, type(self)))
                sys.exit(1)
            Upgrader.tag_set.add(self.tag)
        elif hasattr(self, 'downgrader'):
            if not isinstance(self.downgrader, types.FunctionType):
                print("Error: 'downgrader' for {} is {}, not function".format(
                    self.tag, type(self)))
                sys.exit(1)
            Upgrader.untag_set.add(self.tag)
        else:
            print("Error: no upgrader or downgrader method for {}".format(
                self.tag))
            sys.exit(1)

        if hasattr(self, 'legacy_version'):
            Upgrader.legacy[self.legacy_version] = self

        Upgrader.by_tag[self.tag] = self

    def ready(self, tags):
        for dep in self.depends:
            if dep not in tags:
                return False
        return True

    def update(self, cpt, tags):
        if hasattr(self, 'upgrader'):
            self.upgrader(cpt)
            tags.add(self.tag)
            verboseprint("applied upgrade for", self.tag)
        else:
            self.downgrader(cpt)
            tags.remove(self.tag)
            verboseprint("applied downgrade for", self.tag)

    @staticmethod
    def get(tag):
        return Upgrader.by_tag[tag]

    @staticmethod
    def load_all():
        util_dir = osp.dirname(osp.abspath(__file__))

        for py in glob.glob(util_dir + '/cpt_upgraders/*.py'):
            Upgrader(py)

        # make linear dependences for legacy versions
        i = 3
        while i in Upgrader.legacy:
            Upgrader.legacy[i].depends = [Upgrader.legacy[i-1].tag]
            i = i + 1

        # resolve forward dependencies and audit normal dependencies
        for tag, upg in list(Upgrader.by_tag.items()):
            for fd in upg.fwd_depends:
                if fd not in Upgrader.by_tag:
                    print("Error: '{}' cannot (forward) depend on "
                          "nonexistent tag '{}'".format(fd, tag))
                    sys.exit(1)
                Upgrader.by_tag[fd].depends.append(tag)
            for dep in upg.depends:
                if dep not in Upgrader.by_tag:
                    print("Error: '{}' cannot depend on "
                          "nonexistent tag '{}'".format(tag, dep))
                    sys.exit(1)

def process_file(path, **kwargs):
    if not osp.isfile(path):
        import errno
        raise IOError(errno.ENOENT, "No such file", path)

    verboseprint("Processing file %s...." % path)

    if kwargs.get('backup', True):
        import shutil
        shutil.copyfile(path, path + '.bak')

    cpt = configparser.ConfigParser()

    # gem5 is case sensitive with paramaters
    cpt.optionxform = str

    # Read the current data
    cpt_file = open(path, 'r')
    cpt.read_file(cpt_file)
    cpt_file.close()

    change = False

    # Make sure we know what we're starting from
    if cpt.has_option('root','cpt_ver'):
        cpt_ver = cpt.getint('root','cpt_ver')

        # Legacy linear checkpoint version
        # convert to list of tags before proceeding
        tags = set([])
        for i in range(2, cpt_ver+1):
            tags.add(Upgrader.legacy[i].tag)
        verboseprint("performed legacy version -> tags conversion")
        change = True

        cpt.remove_option('root', 'cpt_ver')
    # @todo The 'Globals' option is deprecated, and should be removed in the
    # future
    elif cpt.has_option('Globals','version_tags'):
        tags = set((''.join(cpt.get('Globals','version_tags'))).split())
    elif cpt.has_option('root.globals','version_tags'):
        tags = set((''.join(cpt.get('root.globals','version_tags'))).split())
    else:
        print("fatal: no version information in checkpoint")
        exit(1)

    verboseprint("has tags", ' '.join(tags))
    # If the current checkpoint has a tag we don't know about, we have
    # a divergence that (in general) must be addressed by (e.g.) merging
    # simulator support for its changes.
    unknown_tags = tags - (Upgrader.tag_set | Upgrader.untag_set)
    if unknown_tags:
        print("warning: upgrade script does not recognize the following "
              "tags in this checkpoint:", ' '.join(unknown_tags))

    # Apply migrations for tags not in checkpoint and tags present for which
    # downgraders are present, respecting dependences
    to_apply = (Upgrader.tag_set - tags) | (Upgrader.untag_set & tags)
    while to_apply:
        ready = set([ t for t in to_apply if Upgrader.get(t).ready(tags) ])
        if not ready:
            print("could not apply these upgrades:", ' '.join(to_apply))
            print("update dependences impossible to resolve; aborting")
            exit(1)

        for tag in ready:
            Upgrader.get(tag).update(cpt, tags)
            change = True

        to_apply -= ready

    if not change:
        verboseprint("...nothing to do")
        return

    cpt.set('root.globals', 'version_tags', ' '.join(tags))

    # Write the old data back
    verboseprint("...completed")
    cpt.write(open(path, 'w'))

if __name__ == '__main__':
    from argparse import ArgumentParser, SUPPRESS
    parser = ArgumentParser(usage="%(prog)s [args] <filename or directory>")
    parser.add_argument(
        "-r", "--recurse", action="store_true",
        help="Recurse through all subdirectories modifying "\
             "each checkpoint that is found")
    parser.add_argument(
        "-N", "--no-backup", action="store_false",
        dest="backup", default=True,
        help="Do no backup each checkpoint before modifying it")
    parser.add_argument(
        "-v", "--verbose", action="store_true",
        help="Print out debugging information as")
    parser.add_argument(
        "--get-cc-file", action="store_true",
        # used during build; generate src/sim/tags.cc and exit
        help=SUPPRESS)
    parser.add_argument("checkpoint", nargs='?')

    args = parser.parse_args()
    verbose_print = args.verbose

    Upgrader.load_all()

    if args.get_cc_file:
        print("// this file is auto-generated by util/cpt_upgrader.py")
        print("#include <string>")
        print("#include <set>")
        print()
        print("namespace gem5")
        print("{")
        print()
        print("std::set<std::string> version_tags = {")
        for tag in Upgrader.tag_set:
            print("  \"{}\",".format(tag))
        print("};")
        print()
        print("} // namespace gem5")
        exit(0)
    elif not args.checkpoint:
        parser.error("You must specify a checkpoint file to modify or a "
                     "directory of checkpoints to recursively update")

    # Deal with shell variables and ~
    path = osp.expandvars(osp.expanduser(args.checkpoint))

    # Process a single file if we have it
    if osp.isfile(path):
        process_file(path, **vars(args))
    # Process an entire directory
    elif osp.isdir(path):
        cpt_file = osp.join(path, 'm5.cpt')
        if args.recurse:
            # Visit very file and see if it matches
            for root,dirs,files in os.walk(path):
                for name in files:
                    if name == 'm5.cpt':
                        process_file(osp.join(root,name), **vars(args))
                for dir in dirs:
                    pass
        # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt
        elif osp.isfile(cpt_file):
            process_file(cpt_file, **vars(args))
        else:
            print("Error: checkpoint file not found in {} ".format(path))
            print("and recurse not specified")
            sys.exit(1)
    sys.exit(0)