avionic design with actual uboot and tooling

submodule of avionic design uboot bootloader and with included tools to
get you started , read readme.md and readme-tk1-loader.md

u-boot/tools/buildman/.gitignore

@@ -0,0 +1 @@
+*.pyc

u-boot/tools/buildman/board.py

@@ -0,0 +1,291 @@
+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# SPDX-License-Identifier:	GPL-2.0+
+#
+
+import re
+
+class Expr:
+    """A single regular expression for matching boards to build"""
+
+    def __init__(self, expr):
+        """Set up a new Expr object.
+
+        Args:
+            expr: String cotaining regular expression to store
+        """
+        self._expr = expr
+        self._re = re.compile(expr)
+
+    def Matches(self, props):
+        """Check if any of the properties match the regular expression.
+
+        Args:
+           props: List of properties to check
+        Returns:
+           True if any of the properties match the regular expression
+        """
+        for prop in props:
+            if self._re.match(prop):
+                return True
+        return False
+
+    def __str__(self):
+        return self._expr
+
+class Term:
+    """A list of expressions each of which must match with properties.
+
+    This provides a list of 'AND' expressions, meaning that each must
+    match the board properties for that board to be built.
+    """
+    def __init__(self):
+        self._expr_list = []
+        self._board_count = 0
+
+    def AddExpr(self, expr):
+        """Add an Expr object to the list to check.
+
+        Args:
+            expr: New Expr object to add to the list of those that must
+                  match for a board to be built.
+        """
+        self._expr_list.append(Expr(expr))
+
+    def __str__(self):
+        """Return some sort of useful string describing the term"""
+        return '&'.join([str(expr) for expr in self._expr_list])
+
+    def Matches(self, props):
+        """Check if any of the properties match this term
+
+        Each of the expressions in the term is checked. All must match.
+
+        Args:
+           props: List of properties to check
+        Returns:
+           True if all of the expressions in the Term match, else False
+        """
+        for expr in self._expr_list:
+            if not expr.Matches(props):
+                return False
+        return True
+
+class Board:
+    """A particular board that we can build"""
+    def __init__(self, status, arch, cpu, soc, vendor, board_name, target, options):
+        """Create a new board type.
+
+        Args:
+            status: define whether the board is 'Active' or 'Orphaned'
+            arch: Architecture name (e.g. arm)
+            cpu: Cpu name (e.g. arm1136)
+            soc: Name of SOC, or '' if none (e.g. mx31)
+            vendor: Name of vendor (e.g. armltd)
+            board_name: Name of board (e.g. integrator)
+            target: Target name (use make <target>_defconfig to configure)
+            options: board-specific options (e.g. integratorcp:CM1136)
+        """
+        self.target = target
+        self.arch = arch
+        self.cpu = cpu
+        self.board_name = board_name
+        self.vendor = vendor
+        self.soc = soc
+        self.props = [self.target, self.arch, self.cpu, self.board_name,
+                      self.vendor, self.soc]
+        self.options = options
+        self.build_it = False
+
+
+class Boards:
+    """Manage a list of boards."""
+    def __init__(self):
+        # Use a simple list here, sinc OrderedDict requires Python 2.7
+        self._boards = []
+
+    def AddBoard(self, board):
+        """Add a new board to the list.
+
+        The board's target member must not already exist in the board list.
+
+        Args:
+            board: board to add
+        """
+        self._boards.append(board)
+
+    def ReadBoards(self, fname):
+        """Read a list of boards from a board file.
+
+        Create a board object for each and add it to our _boards list.
+
+        Args:
+            fname: Filename of boards.cfg file
+        """
+        with open(fname, 'r') as fd:
+            for line in fd:
+                if line[0] == '#':
+                    continue
+                fields = line.split()
+                if not fields:
+                    continue
+                for upto in range(len(fields)):
+                    if fields[upto] == '-':
+                        fields[upto] = ''
+                while len(fields) < 8:
+                    fields.append('')
+                if len(fields) > 8:
+                    fields = fields[:8]
+
+                board = Board(*fields)
+                self.AddBoard(board)
+
+
+    def GetList(self):
+        """Return a list of available boards.
+
+        Returns:
+            List of Board objects
+        """
+        return self._boards
+
+    def GetDict(self):
+        """Build a dictionary containing all the boards.
+
+        Returns:
+            Dictionary:
+                key is board.target
+                value is board
+        """
+        board_dict = {}
+        for board in self._boards:
+            board_dict[board.target] = board
+        return board_dict
+
+    def GetSelectedDict(self):
+        """Return a dictionary containing the selected boards
+
+        Returns:
+            List of Board objects that are marked selected
+        """
+        board_dict = {}
+        for board in self._boards:
+            if board.build_it:
+                board_dict[board.target] = board
+        return board_dict
+
+    def GetSelected(self):
+        """Return a list of selected boards
+
+        Returns:
+            List of Board objects that are marked selected
+        """
+        return [board for board in self._boards if board.build_it]
+
+    def GetSelectedNames(self):
+        """Return a list of selected boards
+
+        Returns:
+            List of board names that are marked selected
+        """
+        return [board.target for board in self._boards if board.build_it]
+
+    def _BuildTerms(self, args):
+        """Convert command line arguments to a list of terms.
+
+        This deals with parsing of the arguments. It handles the '&'
+        operator, which joins several expressions into a single Term.
+
+        For example:
+            ['arm & freescale sandbox', 'tegra']
+
+        will produce 3 Terms containing expressions as follows:
+            arm, freescale
+            sandbox
+            tegra
+
+        The first Term has two expressions, both of which must match for
+        a board to be selected.
+
+        Args:
+            args: List of command line arguments
+        Returns:
+            A list of Term objects
+        """
+        syms = []
+        for arg in args:
+            for word in arg.split():
+                sym_build = []
+                for term in word.split('&'):
+                    if term:
+                        sym_build.append(term)
+                    sym_build.append('&')
+                syms += sym_build[:-1]
+        terms = []
+        term = None
+        oper = None
+        for sym in syms:
+            if sym == '&':
+                oper = sym
+            elif oper:
+                term.AddExpr(sym)
+                oper = None
+            else:
+                if term:
+                    terms.append(term)
+                term = Term()
+                term.AddExpr(sym)
+        if term:
+            terms.append(term)
+        return terms
+
+    def SelectBoards(self, args, exclude=[]):
+        """Mark boards selected based on args
+
+        Args:
+            args: List of strings specifying boards to include, either named,
+                  or by their target, architecture, cpu, vendor or soc. If
+                  empty, all boards are selected.
+            exclude: List of boards to exclude, regardless of 'args'
+
+        Returns:
+            Dictionary which holds the number of boards which were selected
+            due to each argument, arranged by argument.
+        """
+        result = {}
+        terms = self._BuildTerms(args)
+
+        result['all'] = 0
+        for term in terms:
+            result[str(term)] = 0
+
+        exclude_list = []
+        for expr in exclude:
+            exclude_list.append(Expr(expr))
+
+        for board in self._boards:
+            matching_term = None
+            build_it = False
+            if terms:
+                match = False
+                for term in terms:
+                    if term.Matches(board.props):
+                        matching_term = str(term)
+                        build_it = True
+                        break
+            else:
+                build_it = True
+
+            # Check that it is not specifically excluded
+            for expr in exclude_list:
+                if expr.Matches(board.props):
+                    build_it = False
+                    break
+
+            if build_it:
+                board.build_it = True
+                if matching_term:
+                    result[matching_term] += 1
+                result['all'] += 1
+
+        return result

u-boot/tools/buildman/bsettings.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# SPDX-License-Identifier:	GPL-2.0+
+#
+
+import ConfigParser
+import os
+import StringIO
+
+
+def Setup(fname=''):
+    """Set up the buildman settings module by reading config files
+
+    Args:
+        config_fname:   Config filename to read ('' for default)
+    """
+    global settings
+    global config_fname
+
+    settings = ConfigParser.SafeConfigParser()
+    if fname is not None:
+        config_fname = fname
+        if config_fname == '':
+            config_fname = '%s/.buildman' % os.getenv('HOME')
+        if config_fname:
+            settings.read(config_fname)
+
+def AddFile(data):
+    settings.readfp(StringIO.StringIO(data))
+
+def GetItems(section):
+    """Get the items from a section of the config.
+
+    Args:
+        section: name of section to retrieve
+
+    Returns:
+        List of (name, value) tuples for the section
+    """
+    try:
+        return settings.items(section)
+    except ConfigParser.NoSectionError as e:
+        return []
+    except:
+        raise
+
+def SetItem(section, tag, value):
+    """Set an item and write it back to the settings file"""
+    global settings
+    global config_fname
+
+    settings.set(section, tag, value)
+    if config_fname is not None:
+        with open(config_fname, 'w') as fd:
+            settings.write(fd)

u-boot/tools/buildman/builderthread.py

@@ -0,0 +1,486 @@
+# Copyright (c) 2014 Google, Inc
+#
+# SPDX-License-Identifier:      GPL-2.0+
+#
+
+import errno
+import glob
+import os
+import shutil
+import threading
+
+import command
+import gitutil
+
+RETURN_CODE_RETRY = -1
+
+def Mkdir(dirname, parents = False):
+    """Make a directory if it doesn't already exist.
+
+    Args:
+        dirname: Directory to create
+    """
+    try:
+        if parents:
+            os.makedirs(dirname)
+        else:
+            os.mkdir(dirname)
+    except OSError as err:
+        if err.errno == errno.EEXIST:
+            pass
+        else:
+            raise
+
+class BuilderJob:
+    """Holds information about a job to be performed by a thread
+
+    Members:
+        board: Board object to build
+        commits: List of commit options to build.
+    """
+    def __init__(self):
+        self.board = None
+        self.commits = []
+
+
+class ResultThread(threading.Thread):
+    """This thread processes results from builder threads.
+
+    It simply passes the results on to the builder. There is only one
+    result thread, and this helps to serialise the build output.
+    """
+    def __init__(self, builder):
+        """Set up a new result thread
+
+        Args:
+            builder: Builder which will be sent each result
+        """
+        threading.Thread.__init__(self)
+        self.builder = builder
+
+    def run(self):
+        """Called to start up the result thread.
+
+        We collect the next result job and pass it on to the build.
+        """
+        while True:
+            result = self.builder.out_queue.get()
+            self.builder.ProcessResult(result)
+            self.builder.out_queue.task_done()
+
+
+class BuilderThread(threading.Thread):
+    """This thread builds U-Boot for a particular board.
+
+    An input queue provides each new job. We run 'make' to build U-Boot
+    and then pass the results on to the output queue.
+
+    Members:
+        builder: The builder which contains information we might need
+        thread_num: Our thread number (0-n-1), used to decide on a
+                temporary directory
+    """
+    def __init__(self, builder, thread_num, incremental, per_board_out_dir):
+        """Set up a new builder thread"""
+        threading.Thread.__init__(self)
+        self.builder = builder
+        self.thread_num = thread_num
+        self.incremental = incremental
+        self.per_board_out_dir = per_board_out_dir
+
+    def Make(self, commit, brd, stage, cwd, *args, **kwargs):
+        """Run 'make' on a particular commit and board.
+
+        The source code will already be checked out, so the 'commit'
+        argument is only for information.
+
+        Args:
+            commit: Commit object that is being built
+            brd: Board object that is being built
+            stage: Stage of the build. Valid stages are:
+                        mrproper - can be called to clean source
+                        config - called to configure for a board
+                        build - the main make invocation - it does the build
+            args: A list of arguments to pass to 'make'
+            kwargs: A list of keyword arguments to pass to command.RunPipe()
+
+        Returns:
+            CommandResult object
+        """
+        return self.builder.do_make(commit, brd, stage, cwd, *args,
+                **kwargs)
+
+    def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
+                  force_build_failures):
+        """Build a particular commit.
+
+        If the build is already done, and we are not forcing a build, we skip
+        the build and just return the previously-saved results.
+
+        Args:
+            commit_upto: Commit number to build (0...n-1)
+            brd: Board object to build
+            work_dir: Directory to which the source will be checked out
+            do_config: True to run a make <board>_defconfig on the source
+            force_build: Force a build even if one was previously done
+            force_build_failures: Force a bulid if the previous result showed
+                failure
+
+        Returns:
+            tuple containing:
+                - CommandResult object containing the results of the build
+                - boolean indicating whether 'make config' is still needed
+        """
+        # Create a default result - it will be overwritte by the call to
+        # self.Make() below, in the event that we do a build.
+        result = command.CommandResult()
+        result.return_code = 0
+        if self.builder.in_tree:
+            out_dir = work_dir
+        else:
+            if self.per_board_out_dir:
+                out_rel_dir = os.path.join('..', brd.target)
+            else:
+                out_rel_dir = 'build'
+            out_dir = os.path.join(work_dir, out_rel_dir)
+
+        # Check if the job was already completed last time
+        done_file = self.builder.GetDoneFile(commit_upto, brd.target)
+        result.already_done = os.path.exists(done_file)
+        will_build = (force_build or force_build_failures or
+            not result.already_done)
+        if result.already_done:
+            # Get the return code from that build and use it
+            with open(done_file, 'r') as fd:
+                result.return_code = int(fd.readline())
+
+            # Check the signal that the build needs to be retried
+            if result.return_code == RETURN_CODE_RETRY:
+                will_build = True
+            elif will_build:
+                err_file = self.builder.GetErrFile(commit_upto, brd.target)
+                if os.path.exists(err_file) and os.stat(err_file).st_size:
+                    result.stderr = 'bad'
+                elif not force_build:
+                    # The build passed, so no need to build it again
+                    will_build = False
+
+        if will_build:
+            # We are going to have to build it. First, get a toolchain
+            if not self.toolchain:
+                try:
+                    self.toolchain = self.builder.toolchains.Select(brd.arch)
+                except ValueError as err:
+                    result.return_code = 10
+                    result.stdout = ''
+                    result.stderr = str(err)
+                    # TODO(sjg@chromium.org): This gets swallowed, but needs
+                    # to be reported.
+
+            if self.toolchain:
+                # Checkout the right commit
+                if self.builder.commits:
+                    commit = self.builder.commits[commit_upto]
+                    if self.builder.checkout:
+                        git_dir = os.path.join(work_dir, '.git')
+                        gitutil.Checkout(commit.hash, git_dir, work_dir,
+                                         force=True)
+                else:
+                    commit = 'current'
+
+                # Set up the environment and command line
+                env = self.toolchain.MakeEnvironment(self.builder.full_path)
+                Mkdir(out_dir)
+                args = []
+                cwd = work_dir
+                src_dir = os.path.realpath(work_dir)
+                if not self.builder.in_tree:
+                    if commit_upto is None:
+                        # In this case we are building in the original source
+                        # directory (i.e. the current directory where buildman
+                        # is invoked. The output directory is set to this
+                        # thread's selected work directory.
+                        #
+                        # Symlinks can confuse U-Boot's Makefile since
+                        # we may use '..' in our path, so remove them.
+                        out_dir = os.path.realpath(out_dir)
+                        args.append('O=%s' % out_dir)
+                        cwd = None
+                        src_dir = os.getcwd()
+                    else:
+                        args.append('O=%s' % out_rel_dir)
+                if self.builder.verbose_build:
+                    args.append('V=1')
+                else:
+                    args.append('-s')
+                if self.builder.num_jobs is not None:
+                    args.extend(['-j', str(self.builder.num_jobs)])
+                config_args = ['%s_defconfig' % brd.target]
+                config_out = ''
+                args.extend(self.builder.toolchains.GetMakeArguments(brd))
+
+                # If we need to reconfigure, do that now
+                if do_config:
+                    config_out = ''
+                    if not self.incremental:
+                        result = self.Make(commit, brd, 'mrproper', cwd,
+                                'mrproper', *args, env=env)
+                        config_out += result.combined
+                    result = self.Make(commit, brd, 'config', cwd,
+                            *(args + config_args), env=env)
+                    config_out += result.combined
+                    do_config = False   # No need to configure next time
+                if result.return_code == 0:
+                    result = self.Make(commit, brd, 'build', cwd, *args,
+                            env=env)
+                result.stderr = result.stderr.replace(src_dir + '/', '')
+                if self.builder.verbose_build:
+                    result.stdout = config_out + result.stdout
+            else:
+                result.return_code = 1
+                result.stderr = 'No tool chain for %s\n' % brd.arch
+            result.already_done = False
+
+        result.toolchain = self.toolchain
+        result.brd = brd
+        result.commit_upto = commit_upto
+        result.out_dir = out_dir
+        return result, do_config
+
+    def _WriteResult(self, result, keep_outputs):
+        """Write a built result to the output directory.
+
+        Args:
+            result: CommandResult object containing result to write
+            keep_outputs: True to store the output binaries, False
+                to delete them
+        """
+        # Fatal error
+        if result.return_code < 0:
+            return
+
+        # If we think this might have been aborted with Ctrl-C, record the
+        # failure but not that we are 'done' with this board. A retry may fix
+        # it.
+        maybe_aborted =  result.stderr and 'No child processes' in result.stderr
+
+        if result.already_done:
+            return
+
+        # Write the output and stderr
+        output_dir = self.builder._GetOutputDir(result.commit_upto)
+        Mkdir(output_dir)
+        build_dir = self.builder.GetBuildDir(result.commit_upto,
+                result.brd.target)
+        Mkdir(build_dir)
+
+        outfile = os.path.join(build_dir, 'log')
+        with open(outfile, 'w') as fd:
+            if result.stdout:
+                fd.write(result.stdout)
+
+        errfile = self.builder.GetErrFile(result.commit_upto,
+                result.brd.target)
+        if result.stderr:
+            with open(errfile, 'w') as fd:
+                fd.write(result.stderr)
+        elif os.path.exists(errfile):
+            os.remove(errfile)
+
+        if result.toolchain:
+            # Write the build result and toolchain information.
+            done_file = self.builder.GetDoneFile(result.commit_upto,
+                    result.brd.target)
+            with open(done_file, 'w') as fd:
+                if maybe_aborted:
+                    # Special code to indicate we need to retry
+                    fd.write('%s' % RETURN_CODE_RETRY)
+                else:
+                    fd.write('%s' % result.return_code)
+            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
+                print >>fd, 'gcc', result.toolchain.gcc
+                print >>fd, 'path', result.toolchain.path
+                print >>fd, 'cross', result.toolchain.cross
+                print >>fd, 'arch', result.toolchain.arch
+                fd.write('%s' % result.return_code)
+
+            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
+                print >>fd, 'gcc', result.toolchain.gcc
+                print >>fd, 'path', result.toolchain.path
+
+            # Write out the image and function size information and an objdump
+            env = result.toolchain.MakeEnvironment(self.builder.full_path)
+            lines = []
+            for fname in ['u-boot', 'spl/u-boot-spl']:
+                cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
+                nm_result = command.RunPipe([cmd], capture=True,
+                        capture_stderr=True, cwd=result.out_dir,
+                        raise_on_error=False, env=env)
+                if nm_result.stdout:
+                    nm = self.builder.GetFuncSizesFile(result.commit_upto,
+                                    result.brd.target, fname)
+                    with open(nm, 'w') as fd:
+                        print >>fd, nm_result.stdout,
+
+                cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
+                dump_result = command.RunPipe([cmd], capture=True,
+                        capture_stderr=True, cwd=result.out_dir,
+                        raise_on_error=False, env=env)
+                rodata_size = ''
+                if dump_result.stdout:
+                    objdump = self.builder.GetObjdumpFile(result.commit_upto,
+                                    result.brd.target, fname)
+                    with open(objdump, 'w') as fd:
+                        print >>fd, dump_result.stdout,
+                    for line in dump_result.stdout.splitlines():
+                        fields = line.split()
+                        if len(fields) > 5 and fields[1] == '.rodata':
+                            rodata_size = fields[2]
+
+                cmd = ['%ssize' % self.toolchain.cross, fname]
+                size_result = command.RunPipe([cmd], capture=True,
+                        capture_stderr=True, cwd=result.out_dir,
+                        raise_on_error=False, env=env)
+                if size_result.stdout:
+                    lines.append(size_result.stdout.splitlines()[1] + ' ' +
+                                 rodata_size)
+
+            # Write out the image sizes file. This is similar to the output
+            # of binutil's 'size' utility, but it omits the header line and
+            # adds an additional hex value at the end of each line for the
+            # rodata size
+            if len(lines):
+                sizes = self.builder.GetSizesFile(result.commit_upto,
+                                result.brd.target)
+                with open(sizes, 'w') as fd:
+                    print >>fd, '\n'.join(lines)
+
+        # Write out the configuration files, with a special case for SPL
+        for dirname in ['', 'spl', 'tpl']:
+            self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
+                'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
+                'include/autoconf.mk', 'include/generated/autoconf.h'])
+
+        # Now write the actual build output
+        if keep_outputs:
+            self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
+                '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
+                'spl/u-boot-spl*'])
+
+    def CopyFiles(self, out_dir, build_dir, dirname, patterns):
+        """Copy files from the build directory to the output.
+
+        Args:
+            out_dir: Path to output directory containing the files
+            build_dir: Place to copy the files
+            dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
+            patterns: A list of filenames (strings) to copy, each relative
+               to the build directory
+        """
+        for pattern in patterns:
+            file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
+            for fname in file_list:
+                target = os.path.basename(fname)
+                if dirname:
+                    base, ext = os.path.splitext(target)
+                    if ext:
+                        target = '%s-%s%s' % (base, dirname, ext)
+                shutil.copy(fname, os.path.join(build_dir, target))
+
+    def RunJob(self, job):
+        """Run a single job
+
+        A job consists of a building a list of commits for a particular board.
+
+        Args:
+            job: Job to build
+        """
+        brd = job.board
+        work_dir = self.builder.GetThreadDir(self.thread_num)
+        self.toolchain = None
+        if job.commits:
+            # Run 'make board_defconfig' on the first commit
+            do_config = True
+            commit_upto  = 0
+            force_build = False
+            for commit_upto in range(0, len(job.commits), job.step):
+                result, request_config = self.RunCommit(commit_upto, brd,
+                        work_dir, do_config,
+                        force_build or self.builder.force_build,
+                        self.builder.force_build_failures)
+                failed = result.return_code or result.stderr
+                did_config = do_config
+                if failed and not do_config:
+                    # If our incremental build failed, try building again
+                    # with a reconfig.
+                    if self.builder.force_config_on_failure:
+                        result, request_config = self.RunCommit(commit_upto,
+                            brd, work_dir, True, True, False)
+                        did_config = True
+                if not self.builder.force_reconfig:
+                    do_config = request_config
+
+                # If we built that commit, then config is done. But if we got
+                # an warning, reconfig next time to force it to build the same
+                # files that created warnings this time. Otherwise an
+                # incremental build may not build the same file, and we will
+                # think that the warning has gone away.
+                # We could avoid this by using -Werror everywhere...
+                # For errors, the problem doesn't happen, since presumably
+                # the build stopped and didn't generate output, so will retry
+                # that file next time. So we could detect warnings and deal
+                # with them specially here. For now, we just reconfigure if
+                # anything goes work.
+                # Of course this is substantially slower if there are build
+                # errors/warnings (e.g. 2-3x slower even if only 10% of builds
+                # have problems).
+                if (failed and not result.already_done and not did_config and
+                        self.builder.force_config_on_failure):
+                    # If this build failed, try the next one with a
+                    # reconfigure.
+                    # Sometimes if the board_config.h file changes it can mess
+                    # with dependencies, and we get:
+                    # make: *** No rule to make target `include/autoconf.mk',
+                    #     needed by `depend'.
+                    do_config = True
+                    force_build = True
+                else:
+                    force_build = False
+                    if self.builder.force_config_on_failure:
+                        if failed:
+                            do_config = True
+                    result.commit_upto = commit_upto
+                    if result.return_code < 0:
+                        raise ValueError('Interrupt')
+
+                # We have the build results, so output the result
+                self._WriteResult(result, job.keep_outputs)
+                self.builder.out_queue.put(result)
+        else:
+            # Just build the currently checked-out build
+            result, request_config = self.RunCommit(None, brd, work_dir, True,
+                        True, self.builder.force_build_failures)
+            result.commit_upto = 0
+            self._WriteResult(result, job.keep_outputs)
+            self.builder.out_queue.put(result)
+
+    def run(self):
+        """Our thread's run function
+
+        This thread picks a job from the queue, runs it, and then goes to the
+        next job.
+        """
+        alive = True
+        while True:
+            job = self.builder.queue.get()
+            if self.builder.active and alive:
+                self.RunJob(job)
+            '''
+            try:
+                if self.builder.active and alive:
+                    self.RunJob(job)
+            except Exception as err:
+                alive = False
+                print err
+            '''
+            self.builder.queue.task_done()

u-boot/tools/buildman/buildman

@@ -0,0 +1 @@
+buildman.py

u-boot/tools/buildman/buildman.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# SPDX-License-Identifier:	GPL-2.0+
+#
+
+"""See README for more information"""
+
+import multiprocessing
+import os
+import re
+import sys
+import unittest
+
+# Bring in the patman libraries
+our_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(our_path, '../patman'))
+
+# Our modules
+import board
+import bsettings
+import builder
+import checkpatch
+import cmdline
+import control
+import doctest
+import gitutil
+import patchstream
+import terminal
+import toolchain
+
+def RunTests():
+    import func_test
+    import test
+    import doctest
+
+    result = unittest.TestResult()
+    for module in ['toolchain', 'gitutil']:
+        suite = doctest.DocTestSuite(module)
+        suite.run(result)
+
+    sys.argv = [sys.argv[0]]
+    for module in (test.TestBuild, func_test.TestFunctional):
+        suite = unittest.TestLoader().loadTestsFromTestCase(module)
+        suite.run(result)
+
+    print result
+    for test, err in result.errors:
+        print err
+    for test, err in result.failures:
+        print err
+
+
+options, args = cmdline.ParseArgs()
+
+# Run our meagre tests
+if options.test:
+    RunTests()
+
+# Build selected commits for selected boards
+else:
+    bsettings.Setup(options.config_file)
+    ret_code = control.DoBuildman(options, args)
+    sys.exit(ret_code)

u-boot/tools/buildman/cmdline.py

@@ -0,0 +1,101 @@
+#
+# Copyright (c) 2014 Google, Inc
+#
+# SPDX-License-Identifier:      GPL-2.0+
+#
+
+from optparse import OptionParser
+
+def ParseArgs():
+    """Parse command line arguments from sys.argv[]
+
+    Returns:
+        tuple containing:
+            options: command line options
+            args: command lin arguments
+    """
+    parser = OptionParser()
+    parser.add_option('-b', '--branch', type='string',
+          help='Branch name to build, or range of commits to build')
+    parser.add_option('-B', '--bloat', dest='show_bloat',
+          action='store_true', default=False,
+          help='Show changes in function code size for each board')
+    parser.add_option('-c', '--count', dest='count', type='int',
+          default=-1, help='Run build on the top n commits')
+    parser.add_option('-C', '--force-reconfig', dest='force_reconfig',
+          action='store_true', default=False,
+          help='Reconfigure for every commit (disable incremental build)')
+    parser.add_option('-d', '--detail', dest='show_detail',
+          action='store_true', default=False,
+          help='Show detailed information for each board in summary')
+    parser.add_option('-e', '--show_errors', action='store_true',
+          default=False, help='Show errors and warnings')
+    parser.add_option('-f', '--force-build', dest='force_build',
+          action='store_true', default=False,
+          help='Force build of boards even if already built')
+    parser.add_option('-F', '--force-build-failures', dest='force_build_failures',
+          action='store_true', default=False,
+          help='Force build of previously-failed build')
+    parser.add_option('--fetch-arch', type='string',
+          help="Fetch a toolchain for architecture FETCH_ARCH ('list' to list)."
+              ' You can also fetch several toolchains separate by comma, or'
+              " 'all' to download all")
+    parser.add_option('-g', '--git', type='string',
+          help='Git repo containing branch to build', default='.')
+    parser.add_option('-G', '--config-file', type='string',
+          help='Path to buildman config file', default='')
+    parser.add_option('-H', '--full-help', action='store_true', dest='full_help',
+          default=False, help='Display the README file')
+    parser.add_option('-i', '--in-tree', dest='in_tree',
+          action='store_true', default=False,
+          help='Build in the source tree instead of a separate directory')
+    parser.add_option('-I', '--incremental', action='store_true',
+          default=False, help='Do not run make mrproper (when reconfiguring)')
+    parser.add_option('-j', '--jobs', dest='jobs', type='int',
+          default=None, help='Number of jobs to run at once (passed to make)')
+    parser.add_option('-k', '--keep-outputs', action='store_true',
+          default=False, help='Keep all build output files (e.g. binaries)')
+    parser.add_option('-K', '--show-config', action='store_true',
+          default=False, help='Show configuration changes in summary (both board config files and Kconfig)')
+    parser.add_option('-l', '--list-error-boards', action='store_true',
+          default=False, help='Show a list of boards next to each error/warning')
+    parser.add_option('--list-tool-chains', action='store_true', default=False,
+          help='List available tool chains')
+    parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run',
+          default=False, help="Do a dry run (describe actions, but do nothing)")
+    parser.add_option('-N', '--no-subdirs', action='store_true', dest='no_subdirs',
+          default=False, help="Don't create subdirectories when building current source for a single board")
+    parser.add_option('-o', '--output-dir', type='string',
+          dest='output_dir', default='..',
+          help='Directory where all builds happen and buildman has its workspace (default is ../)')
+    parser.add_option('-Q', '--quick', action='store_true',
+          default=False, help='Do a rough build, with limited warning resolution')
+    parser.add_option('-p', '--full-path', action='store_true',
+          default=False, help="Use full toolchain path in CROSS_COMPILE")
+    parser.add_option('-P', '--per-board-out-dir', action='store_true',
+          default=False, help="Use an O= (output) directory per board rather than per thread")
+    parser.add_option('-s', '--summary', action='store_true',
+          default=False, help='Show a build summary')
+    parser.add_option('-S', '--show-sizes', action='store_true',
+          default=False, help='Show image size variation in summary')
+    parser.add_option('--step', type='int',
+          default=1, help='Only build every n commits (0=just first and last)')
+    parser.add_option('-t', '--test', action='store_true', dest='test',
+                      default=False, help='run tests')
+    parser.add_option('-T', '--threads', type='int',
+          default=None, help='Number of builder threads to use')
+    parser.add_option('-u', '--show_unknown', action='store_true',
+          default=False, help='Show boards with unknown build result')
+    parser.add_option('-v', '--verbose', action='store_true',
+          default=False, help='Show build results while the build progresses')
+    parser.add_option('-V', '--verbose-build', action='store_true',
+          default=False, help='Run make with V=1, logging all output')
+    parser.add_option('-x', '--exclude', dest='exclude',
+          type='string', action='append',
+          help='Specify a list of boards to exclude, separated by comma')
+
+    parser.usage += """
+
+    Build U-Boot for all commits in a branch. Use -n to do a dry run"""
+
+    return parser.parse_args()

u-boot/tools/buildman/control.py

@@ -0,0 +1,299 @@
+# Copyright (c) 2013 The Chromium OS Authors.
+#
+# SPDX-License-Identifier:	GPL-2.0+
+#
+
+import multiprocessing
+import os
+import shutil
+import sys
+
+import board
+import bsettings
+from builder import Builder
+import gitutil
+import patchstream
+import terminal
+from terminal import Print
+import toolchain
+import command
+import subprocess
+
+def GetPlural(count):
+    """Returns a plural 's' if count is not 1"""
+    return 's' if count != 1 else ''
+
+def GetActionSummary(is_summary, commits, selected, options):
+    """Return a string summarising the intended action.
+
+    Returns:
+        Summary string.
+    """
+    if commits:
+        count = len(commits)
+        count = (count + options.step - 1) / options.step
+        commit_str = '%d commit%s' % (count, GetPlural(count))
+    else:
+        commit_str = 'current source'
+    str = '%s %s for %d boards' % (
+        'Summary of' if is_summary else 'Building', commit_str,
+        len(selected))
+    str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
+            GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
+    return str
+
+def ShowActions(series, why_selected, boards_selected, builder, options):
+    """Display a list of actions that we would take, if not a dry run.
+
+    Args:
+        series: Series object
+        why_selected: Dictionary where each key is a buildman argument
+                provided by the user, and the value is the boards brought
+                in by that argument. For example, 'arm' might bring in
+                400 boards, so in this case the key would be 'arm' and
+                the value would be a list of board names.
+        boards_selected: Dict of selected boards, key is target name,
+                value is Board object
+        builder: The builder that will be used to build the commits
+        options: Command line options object
+    """
+    col = terminal.Color()
+    print 'Dry run, so not doing much. But I would do this:'
+    print
+    if series:
+        commits = series.commits
+    else:
+        commits = None
+    print GetActionSummary(False, commits, boards_selected,
+            options)
+    print 'Build directory: %s' % builder.base_dir
+    if commits:
+        for upto in range(0, len(series.commits), options.step):
+            commit = series.commits[upto]
+            print '   ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
+            print commit.subject
+    print
+    for arg in why_selected:
+        if arg != 'all':
+            print arg, ': %d boards' % why_selected[arg]
+    print ('Total boards to build for each commit: %d\n' %
+            why_selected['all'])
+
+def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
+               clean_dir=False):
+    """The main control code for buildman
+
+    Args:
+        options: Command line options object
+        args: Command line arguments (list of strings)
+        toolchains: Toolchains to use - this should be a Toolchains()
+                object. If None, then it will be created and scanned
+        make_func: Make function to use for the builder. This is called
+                to execute 'make'. If this is None, the normal function
+                will be used, which calls the 'make' tool with suitable
+                arguments. This setting is useful for tests.
+        board: Boards() object to use, containing a list of available
+                boards. If this is None it will be created and scanned.
+    """
+    global builder
+
+    if options.full_help:
+        pager = os.getenv('PAGER')
+        if not pager:
+            pager = 'more'
+        fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
+                             'README')
+        command.Run(pager, fname)
+        return 0
+
+    gitutil.Setup()
+
+    options.git_dir = os.path.join(options.git, '.git')
+
+    if not toolchains:
+        toolchains = toolchain.Toolchains()
+        toolchains.GetSettings()
+        toolchains.Scan(options.list_tool_chains)
+    if options.list_tool_chains:
+        toolchains.List()
+        print
+        return 0
+
+    if options.fetch_arch:
+        if options.fetch_arch == 'list':
+            sorted_list = toolchains.ListArchs()
+            print 'Available architectures: %s\n' % ' '.join(sorted_list)
+            return 0
+        else:
+            fetch_arch = options.fetch_arch
+            if fetch_arch == 'all':
+                fetch_arch = ','.join(toolchains.ListArchs())
+                print 'Downloading toolchains: %s\n' % fetch_arch
+            for arch in fetch_arch.split(','):
+                ret = toolchains.FetchAndInstall(arch)
+                if ret:
+                    return ret
+            return 0
+
+    # Work out how many commits to build. We want to build everything on the
+    # branch. We also build the upstream commit as a control so we can see
+    # problems introduced by the first commit on the branch.
+    col = terminal.Color()
+    count = options.count
+    has_range = options.branch and '..' in options.branch
+    if count == -1:
+        if not options.branch:
+            count = 1
+        else:
+            if has_range:
+                count, msg = gitutil.CountCommitsInRange(options.git_dir,
+                                                         options.branch)
+            else:
+                count, msg = gitutil.CountCommitsInBranch(options.git_dir,
+                                                          options.branch)
+            if count is None:
+                sys.exit(col.Color(col.RED, msg))
+            elif count == 0:
+                sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
+                                   options.branch))
+            if msg:
+                print col.Color(col.YELLOW, msg)
+            count += 1   # Build upstream commit also
+
+    if not count:
+        str = ("No commits found to process in branch '%s': "
+               "set branch's upstream or use -c flag" % options.branch)
+        sys.exit(col.Color(col.RED, str))
+
+    # Work out what subset of the boards we are building
+    if not boards:
+        board_file = os.path.join(options.git, 'boards.cfg')
+        status = subprocess.call([os.path.join(options.git,
+                                                'tools/genboardscfg.py')])
+        if status != 0:
+                sys.exit("Failed to generate boards.cfg")
+
+        boards = board.Boards()
+        boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
+
+    exclude = []
+    if options.exclude:
+        for arg in options.exclude:
+            exclude += arg.split(',')
+
+    why_selected = boards.SelectBoards(args, exclude)
+    selected = boards.GetSelected()
+    if not len(selected):
+        sys.exit(col.Color(col.RED, 'No matching boards found'))
+
+    # Read the metadata from the commits. First look at the upstream commit,
+    # then the ones in the branch. We would like to do something like
+    # upstream/master~..branch but that isn't possible if upstream/master is
+    # a merge commit (it will list all the commits that form part of the
+    # merge)
+    # Conflicting tags are not a problem for buildman, since it does not use
+    # them. For example, Series-version is not useful for buildman. On the
+    # other hand conflicting tags will cause an error. So allow later tags
+    # to overwrite earlier ones by setting allow_overwrite=True
+    if options.branch:
+        if count == -1:
+            if has_range:
+                range_expr = options.branch
+            else:
+                range_expr = gitutil.GetRangeInBranch(options.git_dir,
+                                                      options.branch)
+            upstream_commit = gitutil.GetUpstream(options.git_dir,
+                                                  options.branch)
+            series = patchstream.GetMetaDataForList(upstream_commit,
+                options.git_dir, 1, series=None, allow_overwrite=True)
+
+            series = patchstream.GetMetaDataForList(range_expr,
+                    options.git_dir, None, series, allow_overwrite=True)
+        else:
+            # Honour the count
+            series = patchstream.GetMetaDataForList(options.branch,
+                    options.git_dir, count, series=None, allow_overwrite=True)
+    else:
+        series = None
+        options.verbose = True
+        if not options.summary:
+            options.show_errors = True
+
+    # By default we have one thread per CPU. But if there are not enough jobs
+    # we can have fewer threads and use a high '-j' value for make.
+    if not options.threads:
+        options.threads = min(multiprocessing.cpu_count(), len(selected))
+    if not options.jobs:
+        options.jobs = max(1, (multiprocessing.cpu_count() +
+                len(selected) - 1) / len(selected))
+
+    if not options.step:
+        options.step = len(series.commits) - 1
+
+    gnu_make = command.Output(os.path.join(options.git,
+                                           'scripts/show-gnu-make')).rstrip()
+    if not gnu_make:
+        sys.exit('GNU Make not found')
+
+    # Create a new builder with the selected options.
+    output_dir = options.output_dir
+    if options.branch:
+        dirname = options.branch.replace('/', '_')
+        # As a special case allow the board directory to be placed in the
+        # output directory itself rather than any subdirectory.
+        if not options.no_subdirs:
+            output_dir = os.path.join(options.output_dir, dirname)
+    if (clean_dir and output_dir != options.output_dir and
+            os.path.exists(output_dir)):
+        shutil.rmtree(output_dir)
+    builder = Builder(toolchains, output_dir, options.git_dir,
+            options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
+            show_unknown=options.show_unknown, step=options.step,
+            no_subdirs=options.no_subdirs, full_path=options.full_path,
+            verbose_build=options.verbose_build,
+            incremental=options.incremental,
+            per_board_out_dir=options.per_board_out_dir,)
+    builder.force_config_on_failure = not options.quick
+    if make_func:
+        builder.do_make = make_func
+
+    # For a dry run, just show our actions as a sanity check
+    if options.dry_run:
+        ShowActions(series, why_selected, selected, builder, options)
+    else:
+        builder.force_build = options.force_build
+        builder.force_build_failures = options.force_build_failures
+        builder.force_reconfig = options.force_reconfig
+        builder.in_tree = options.in_tree
+
+        # Work out which boards to build
+        board_selected = boards.GetSelectedDict()
+
+        if series:
+            commits = series.commits
+            # Number the commits for test purposes
+            for commit in range(len(commits)):
+                commits[commit].sequence = commit
+        else:
+            commits = None
+
+        Print(GetActionSummary(options.summary, commits, board_selected,
+                                options))
+
+        # We can't show function sizes without board details at present
+        if options.show_bloat:
+            options.show_detail = True
+        builder.SetDisplayOptions(options.show_errors, options.show_sizes,
+                                  options.show_detail, options.show_bloat,
+                                  options.list_error_boards,
+                                  options.show_config)
+        if options.summary:
+            builder.ShowSummary(commits, board_selected)
+        else:
+            fail, warned = builder.BuildBoards(commits, board_selected,
+                                options.keep_outputs, options.verbose)
+            if fail:
+                return 128
+            elif warned:
+                return 129
+    return 0

u-boot/tools/buildman/func_test.py

@@ -0,0 +1,521 @@
+#
+# Copyright (c) 2014 Google, Inc
+#
+# SPDX-License-Identifier:      GPL-2.0+
+#
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+import board
+import bsettings
+import cmdline
+import command
+import control
+import gitutil
+import terminal
+import toolchain
+
+settings_data = '''
+# Buildman settings file
+
+[toolchain]
+
+[toolchain-alias]
+
+[make-flags]
+src=/home/sjg/c/src
+chroot=/home/sjg/c/chroot
+vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
+chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
+chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
+chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
+'''
+
+boards = [
+    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
+    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
+    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
+    ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
+    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
+]
+
+commit_shortlog = """4aca821 patman: Avoid changing the order of tags
+39403bb patman: Use --no-pager' to stop git from forking a pager
+db6e6f2 patman: Remove the -a option
+f2ccf03 patman: Correct unit tests to run correctly
+1d097f9 patman: Fix indentation in terminal.py
+d073747 patman: Support the 'reverse' option for 'git log
+"""
+
+commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
+Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
+Date:   Fri Aug 22 19:12:41 2014 +0900
+
+    buildman: refactor help message
+
+    "buildman [options]" is displayed by default.
+
+    Append the rest of help messages to parser.usage
+    instead of replacing it.
+
+    Besides, "-b <branch>" is not mandatory since commit fea5858e.
+    Drop it from the usage.
+
+    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
+""",
+"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
+Author: Simon Glass <sjg@chromium.org>
+Date:   Thu Aug 14 16:48:25 2014 -0600
+
+    patman: Support the 'reverse' option for 'git log'
+
+    This option is currently not supported, but needs to be, for buildman to
+    operate as expected.
+
+    Series-changes: 7
+    - Add new patch to fix the 'reverse' bug
+
+    Series-version: 8
+
+    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
+    Reported-by: York Sun <yorksun@freescale.com>
+    Signed-off-by: Simon Glass <sjg@chromium.org>
+
+""",
+"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
+Author: Simon Glass <sjg@chromium.org>
+Date:   Sat Aug 9 11:44:32 2014 -0600
+
+    patman: Fix indentation in terminal.py
+
+    This code came from a different project with 2-character indentation. Fix
+    it for U-Boot.
+
+    Series-changes: 6
+    - Add new patch to fix indentation in teminal.py
+
+    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
+    Signed-off-by: Simon Glass <sjg@chromium.org>
+
+""",
+"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
+Author: Simon Glass <sjg@chromium.org>
+Date:   Sat Aug 9 11:08:24 2014 -0600
+
+    patman: Correct unit tests to run correctly
+
+    It seems that doctest behaves differently now, and some of the unit tests
+    do not run. Adjust the tests to work correctly.
+
+     ./tools/patman/patman --test
+    <unittest.result.TestResult run=10 errors=0 failures=0>
+
+    Series-changes: 6
+    - Add new patch to fix patman unit tests
+
+    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
+
+""",
+"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
+Author: Simon Glass <sjg@chromium.org>
+Date:   Sat Aug 9 12:06:02 2014 -0600
+
+    patman: Remove the -a option
+
+    It seems that this is no longer needed, since checkpatch.pl will catch
+    whitespace problems in patches. Also the option is not widely used, so
+    it seems safe to just remove it.
+
+    Series-changes: 6
+    - Add new patch to remove patman's -a option
+
+    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
+    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
+
+""",
+"""commit 39403bb4f838153028a6f21ca30bf100f3791133
+Author: Simon Glass <sjg@chromium.org>
+Date:   Thu Aug 14 21:50:52 2014 -0600
+
+    patman: Use --no-pager' to stop git from forking a pager
+
+""",
+"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
+Author: Simon Glass <sjg@chromium.org>
+Date:   Fri Aug 22 15:57:39 2014 -0600
+
+    patman: Avoid changing the order of tags
+
+    patman collects tags that it sees in the commit and places them nicely
+    sorted at the end of the patch. However, this is not really necessary and
+    in fact is apparently not desirable.
+
+    Series-changes: 9
+    - Add new patch to avoid changing the order of tags
+
+    Series-version: 9
+
+    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
+    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
+"""]
+
+TEST_BRANCH = '__testbranch'
+
+class TestFunctional(unittest.TestCase):
+    """Functional test for buildman.
+
+    This aims to test from just below the invocation of buildman (parsing
+    of arguments) to 'make' and 'git' invocation. It is not a true
+    emd-to-end test, as it mocks git, make and the tool chain. But this
+    makes it easier to detect when the builder is doing the wrong thing,
+    since in many cases this test code will fail. For example, only a
+    very limited subset of 'git' arguments is supported - anything
+    unexpected will fail.
+    """
+    def setUp(self):
+        self._base_dir = tempfile.mkdtemp()
+        self._git_dir = os.path.join(self._base_dir, 'src')
+        self._buildman_pathname = sys.argv[0]
+        self._buildman_dir = os.path.dirname(sys.argv[0])
+        command.test_result = self._HandleCommand
+        self.setupToolchains()
+        self._toolchains.Add('arm-gcc', test=False)
+        self._toolchains.Add('powerpc-gcc', test=False)
+        bsettings.Setup(None)
+        bsettings.AddFile(settings_data)
+        self._boards = board.Boards()
+        for brd in boards:
+            self._boards.AddBoard(board.Board(*brd))
+
+        # Directories where the source been cloned
+        self._clone_dirs = []
+        self._commits = len(commit_shortlog.splitlines()) + 1
+        self._total_builds = self._commits * len(boards)
+
+        # Number of calls to make
+        self._make_calls = 0
+
+        # Map of [board, commit] to error messages
+        self._error = {}
+
+        self._test_branch = TEST_BRANCH
+
+        # Avoid sending any output and clear all terminal output
+        terminal.SetPrintTestMode()
+        terminal.GetPrintTestLines()
+
+    def tearDown(self):
+        shutil.rmtree(self._base_dir)
+
+    def setupToolchains(self):
+        self._toolchains = toolchain.Toolchains()
+        self._toolchains.Add('gcc', test=False)
+
+    def _RunBuildman(self, *args):
+        return command.RunPipe([[self._buildman_pathname] + list(args)],
+                capture=True, capture_stderr=True)
+
+    def _RunControl(self, *args, **kwargs):
+        sys.argv = [sys.argv[0]] + list(args)
+        options, args = cmdline.ParseArgs()
+        result = control.DoBuildman(options, args, toolchains=self._toolchains,
+                make_func=self._HandleMake, boards=self._boards,
+                clean_dir=kwargs.get('clean_dir', True))
+        self._builder = control.builder
+        return result
+
+    def testFullHelp(self):
+        command.test_result = None
+        result = self._RunBuildman('-H')
+        help_file = os.path.join(self._buildman_dir, 'README')
+        self.assertEqual(len(result.stdout), os.path.getsize(help_file))
+        self.assertEqual(0, len(result.stderr))
+        self.assertEqual(0, result.return_code)
+
+    def testHelp(self):
+        command.test_result = None
+        result = self._RunBuildman('-h')
+        help_file = os.path.join(self._buildman_dir, 'README')
+        self.assertTrue(len(result.stdout) > 1000)
+        self.assertEqual(0, len(result.stderr))
+        self.assertEqual(0, result.return_code)
+
+    def testGitSetup(self):
+        """Test gitutils.Setup(), from outside the module itself"""
+        command.test_result = command.CommandResult(return_code=1)
+        gitutil.Setup()
+        self.assertEqual(gitutil.use_no_decorate, False)
+
+        command.test_result = command.CommandResult(return_code=0)
+        gitutil.Setup()
+        self.assertEqual(gitutil.use_no_decorate, True)
+
+    def _HandleCommandGitLog(self, args):
+        if args[-1] == '--':
+            args = args[:-1]
+        if '-n0' in args:
+            return command.CommandResult(return_code=0)
+        elif args[-1] == 'upstream/master..%s' % self._test_branch:
+            return command.CommandResult(return_code=0, stdout=commit_shortlog)
+        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
+            if args[-1] == self._test_branch:
+                count = int(args[3][2:])
+                return command.CommandResult(return_code=0,
+                                            stdout=''.join(commit_log[:count]))
+
+        # Not handled, so abort
+        print 'git log', args
+        sys.exit(1)
+
+    def _HandleCommandGitConfig(self, args):
+        config = args[0]
+        if config == 'sendemail.aliasesfile':
+            return command.CommandResult(return_code=0)
+        elif config.startswith('branch.badbranch'):
+            return command.CommandResult(return_code=1)
+        elif config == 'branch.%s.remote' % self._test_branch:
+            return command.CommandResult(return_code=0, stdout='upstream\n')
+        elif config == 'branch.%s.merge' % self._test_branch:
+            return command.CommandResult(return_code=0,
+                                         stdout='refs/heads/master\n')
+
+        # Not handled, so abort
+        print 'git config', args
+        sys.exit(1)
+
+    def _HandleCommandGit(self, in_args):
+        """Handle execution of a git command
+
+        This uses a hacked-up parser.
+
+        Args:
+            in_args: Arguments after 'git' from the command line
+        """
+        git_args = []           # Top-level arguments to git itself
+        sub_cmd = None          # Git sub-command selected
+        args = []               # Arguments to the git sub-command
+        for arg in in_args:
+            if sub_cmd:
+                args.append(arg)
+            elif arg[0] == '-':
+                git_args.append(arg)
+            else:
+                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
+                    git_args.append(arg)
+                else:
+                    sub_cmd = arg
+        if sub_cmd == 'config':
+            return self._HandleCommandGitConfig(args)
+        elif sub_cmd == 'log':
+            return self._HandleCommandGitLog(args)
+        elif sub_cmd == 'clone':
+            return command.CommandResult(return_code=0)
+        elif sub_cmd == 'checkout':
+            return command.CommandResult(return_code=0)
+
+        # Not handled, so abort
+        print 'git', git_args, sub_cmd, args
+        sys.exit(1)
+
+    def _HandleCommandNm(self, args):
+        return command.CommandResult(return_code=0)
+
+    def _HandleCommandObjdump(self, args):
+        return command.CommandResult(return_code=0)
+
+    def _HandleCommandSize(self, args):
+        return command.CommandResult(return_code=0)
+
+    def _HandleCommand(self, **kwargs):
+        """Handle a command execution.
+
+        The command is in kwargs['pipe-list'], as a list of pipes, each a
+        list of commands. The command should be emulated as required for
+        testing purposes.
+
+        Returns:
+            A CommandResult object
+        """
+        pipe_list = kwargs['pipe_list']
+        wc = False
+        if len(pipe_list) != 1:
+            if pipe_list[1] == ['wc', '-l']:
+                wc = True
+            else:
+                print 'invalid pipe', kwargs
+                sys.exit(1)
+        cmd = pipe_list[0][0]
+        args = pipe_list[0][1:]
+        result = None
+        if cmd == 'git':
+            result = self._HandleCommandGit(args)
+        elif cmd == './scripts/show-gnu-make':
+            return command.CommandResult(return_code=0, stdout='make')
+        elif cmd.endswith('nm'):
+            return self._HandleCommandNm(args)
+        elif cmd.endswith('objdump'):
+            return self._HandleCommandObjdump(args)
+        elif cmd.endswith( 'size'):
+            return self._HandleCommandSize(args)
+
+        if not result:
+            # Not handled, so abort
+            print 'unknown command', kwargs
+            sys.exit(1)
+
+        if wc:
+            result.stdout = len(result.stdout.splitlines())
+        return result
+
+    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
+        """Handle execution of 'make'
+
+        Args:
+            commit: Commit object that is being built
+            brd: Board object that is being built
+            stage: Stage that we are at (mrproper, config, build)
+            cwd: Directory where make should be run
+            args: Arguments to pass to make
+            kwargs: Arguments to pass to command.RunPipe()
+        """
+        self._make_calls += 1
+        if stage == 'mrproper':
+            return command.CommandResult(return_code=0)
+        elif stage == 'config':
+            return command.CommandResult(return_code=0,
+                    combined='Test configuration complete')
+        elif stage == 'build':
+            stderr = ''
+            if type(commit) is not str:
+                stderr = self._error.get((brd.target, commit.sequence))
+            if stderr:
+                return command.CommandResult(return_code=1, stderr=stderr)
+            return command.CommandResult(return_code=0)
+
+        # Not handled, so abort
+        print 'make', stage
+        sys.exit(1)
+
+    # Example function to print output lines
+    def print_lines(self, lines):
+        print len(lines)
+        for line in lines:
+            print line
+        #self.print_lines(terminal.GetPrintTestLines())
+
+    def testNoBoards(self):
+        """Test that buildman aborts when there are no boards"""
+        self._boards = board.Boards()
+        with self.assertRaises(SystemExit):
+            self._RunControl()
+
+    def testCurrentSource(self):
+        """Very simple test to invoke buildman on the current source"""
+        self.setupToolchains();
+        self._RunControl()
+        lines = terminal.GetPrintTestLines()
+        self.assertIn('Building current source for %d boards' % len(boards),
+                      lines[0].text)
+
+    def testBadBranch(self):
+        """Test that we can detect an invalid branch"""
+        with self.assertRaises(ValueError):
+            self._RunControl('-b', 'badbranch')
+
+    def testBadToolchain(self):
+        """Test that missing toolchains are detected"""
+        self.setupToolchains();
+        ret_code = self._RunControl('-b', TEST_BRANCH)
+        lines = terminal.GetPrintTestLines()
+
+        # Buildman always builds the upstream commit as well
+        self.assertIn('Building %d commits for %d boards' %
+                (self._commits, len(boards)), lines[0].text)
+        self.assertEqual(self._builder.count, self._total_builds)
+
+        # Only sandbox should succeed, the others don't have toolchains
+        self.assertEqual(self._builder.fail,
+                         self._total_builds - self._commits)
+        self.assertEqual(ret_code, 128)
+
+        for commit in range(self._commits):
+            for board in self._boards.GetList():
+                if board.arch != 'sandbox':
+                  errfile = self._builder.GetErrFile(commit, board.target)
+                  fd = open(errfile)
+                  self.assertEqual(fd.readlines(),
+                          ['No tool chain for %s\n' % board.arch])
+                  fd.close()
+
+    def testBranch(self):
+        """Test building a branch with all toolchains present"""
+        self._RunControl('-b', TEST_BRANCH)
+        self.assertEqual(self._builder.count, self._total_builds)
+        self.assertEqual(self._builder.fail, 0)
+
+    def testCount(self):
+        """Test building a specific number of commitst"""
+        self._RunControl('-b', TEST_BRANCH, '-c2')
+        self.assertEqual(self._builder.count, 2 * len(boards))
+        self.assertEqual(self._builder.fail, 0)
+        # Each board has a mrproper, config, and then one make per commit
+        self.assertEqual(self._make_calls, len(boards) * (2 + 2))
+
+    def testIncremental(self):
+        """Test building a branch twice - the second time should do nothing"""
+        self._RunControl('-b', TEST_BRANCH)
+
+        # Each board has a mrproper, config, and then one make per commit
+        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
+        self._make_calls = 0
+        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
+        self.assertEqual(self._make_calls, 0)
+        self.assertEqual(self._builder.count, self._total_builds)
+        self.assertEqual(self._builder.fail, 0)
+
+    def testForceBuild(self):
+        """The -f flag should force a rebuild"""
+        self._RunControl('-b', TEST_BRANCH)
+        self._make_calls = 0
+        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
+        # Each board has a mrproper, config, and then one make per commit
+        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
+
+    def testForceReconfigure(self):
+        """The -f flag should force a rebuild"""
+        self._RunControl('-b', TEST_BRANCH, '-C')
+        # Each commit has a mrproper, config and make
+        self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
+
+    def testErrors(self):
+        """Test handling of build errors"""
+        self._error['board2', 1] = 'fred\n'
+        self._RunControl('-b', TEST_BRANCH)
+        self.assertEqual(self._builder.count, self._total_builds)
+        self.assertEqual(self._builder.fail, 1)
+
+        # Remove the error. This should have no effect since the commit will
+        # not be rebuilt
+        del self._error['board2', 1]
+        self._make_calls = 0
+        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
+        self.assertEqual(self._builder.count, self._total_builds)
+        self.assertEqual(self._make_calls, 0)
+        self.assertEqual(self._builder.fail, 1)
+
+        # Now use the -F flag to force rebuild of the bad commit
+        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
+        self.assertEqual(self._builder.count, self._total_builds)
+        self.assertEqual(self._builder.fail, 0)
+        self.assertEqual(self._make_calls, 3)
+
+    def testBranchWithSlash(self):
+        """Test building a branch with a '/' in the name"""
+        self._test_branch = '/__dev/__testbranch'
+        self._RunControl('-b', self._test_branch, clean_dir=False)
+        self.assertEqual(self._builder.count, self._total_builds)
+        self.assertEqual(self._builder.fail, 0)

u-boot/tools/buildman/test.py

@@ -0,0 +1,419 @@
+#
+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# SPDX-License-Identifier:	GPL-2.0+
+#
+
+import os
+import shutil
+import sys
+import tempfile
+import time
+import unittest
+
+# Bring in the patman libraries
+our_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(our_path, '../patman'))
+
+import board
+import bsettings
+import builder
+import control
+import command
+import commit
+import terminal
+import toolchain
+
+settings_data = '''
+# Buildman settings file
+
+[toolchain]
+main: /usr/sbin
+
+[toolchain-alias]
+x86: i386 x86_64
+'''
+
+errors = [
+    '''main.c: In function 'main_loop':
+main.c:260:6: warning: unused variable 'joe' [-Wunused-variable]
+''',
+    '''main.c: In function 'main_loop2':
+main.c:295:2: error: 'fred' undeclared (first use in this function)
+main.c:295:2: note: each undeclared identifier is reported only once for each function it appears in
+make[1]: *** [main.o] Error 1
+make: *** [common/libcommon.o] Error 2
+Make failed
+''',
+    '''main.c: In function 'main_loop3':
+main.c:280:6: warning: unused variable 'mary' [-Wunused-variable]
+''',
+    '''powerpc-linux-ld: warning: dot moved backwards before `.bss'
+powerpc-linux-ld: warning: dot moved backwards before `.bss'
+powerpc-linux-ld: u-boot: section .text lma 0xfffc0000 overlaps previous sections
+powerpc-linux-ld: u-boot: section .rodata lma 0xfffef3ec overlaps previous sections
+powerpc-linux-ld: u-boot: section .reloc lma 0xffffa400 overlaps previous sections
+powerpc-linux-ld: u-boot: section .data lma 0xffffcd38 overlaps previous sections
+powerpc-linux-ld: u-boot: section .u_boot_cmd lma 0xffffeb40 overlaps previous sections
+powerpc-linux-ld: u-boot: section .bootpg lma 0xfffff198 overlaps previous sections
+''',
+   '''In file included from %(basedir)sarch/sandbox/cpu/cpu.c:9:0:
+%(basedir)sarch/sandbox/include/asm/state.h:44:0: warning: "xxxx" redefined [enabled by default]
+%(basedir)sarch/sandbox/include/asm/state.h:43:0: note: this is the location of the previous definition
+%(basedir)sarch/sandbox/cpu/cpu.c: In function 'do_reset':
+%(basedir)sarch/sandbox/cpu/cpu.c:27:1: error: unknown type name 'blah'
+%(basedir)sarch/sandbox/cpu/cpu.c:28:12: error: expected declaration specifiers or '...' before numeric constant
+make[2]: *** [arch/sandbox/cpu/cpu.o] Error 1
+make[1]: *** [arch/sandbox/cpu] Error 2
+make[1]: *** Waiting for unfinished jobs....
+In file included from %(basedir)scommon/board_f.c:55:0:
+%(basedir)sarch/sandbox/include/asm/state.h:44:0: warning: "xxxx" redefined [enabled by default]
+%(basedir)sarch/sandbox/include/asm/state.h:43:0: note: this is the location of the previous definition
+make: *** [sub-make] Error 2
+'''
+]
+
+
+# hash, subject, return code, list of errors/warnings
+commits = [
+    ['1234', 'upstream/master, ok', 0, []],
+    ['5678', 'Second commit, a warning', 0, errors[0:1]],
+    ['9012', 'Third commit, error', 1, errors[0:2]],
+    ['3456', 'Fourth commit, warning', 0, [errors[0], errors[2]]],
+    ['7890', 'Fifth commit, link errors', 1, [errors[0], errors[3]]],
+    ['abcd', 'Sixth commit, fixes all errors', 0, []],
+    ['ef01', 'Seventh commit, check directory suppression', 1, [errors[4]]],
+]
+
+boards = [
+    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
+    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
+    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
+    ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
+    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
+]
+
+BASE_DIR = 'base'
+
+class Options:
+    """Class that holds build options"""
+    pass
+
+class TestBuild(unittest.TestCase):
+    """Test buildman
+
+    TODO: Write tests for the rest of the functionality
+    """
+    def setUp(self):
+        # Set up commits to build
+        self.commits = []
+        sequence = 0
+        for commit_info in commits:
+            comm = commit.Commit(commit_info[0])
+            comm.subject = commit_info[1]
+            comm.return_code = commit_info[2]
+            comm.error_list = commit_info[3]
+            comm.sequence = sequence
+            sequence += 1
+            self.commits.append(comm)
+
+        # Set up boards to build
+        self.boards = board.Boards()
+        for brd in boards:
+            self.boards.AddBoard(board.Board(*brd))
+        self.boards.SelectBoards([])
+
+        # Add some test settings
+        bsettings.Setup(None)
+        bsettings.AddFile(settings_data)
+
+        # Set up the toolchains
+        self.toolchains = toolchain.Toolchains()
+        self.toolchains.Add('arm-linux-gcc', test=False)
+        self.toolchains.Add('sparc-linux-gcc', test=False)
+        self.toolchains.Add('powerpc-linux-gcc', test=False)
+        self.toolchains.Add('gcc', test=False)
+
+        # Avoid sending any output
+        terminal.SetPrintTestMode()
+        self._col = terminal.Color()
+
+    def Make(self, commit, brd, stage, *args, **kwargs):
+        global base_dir
+
+        result = command.CommandResult()
+        boardnum = int(brd.target[-1])
+        result.return_code = 0
+        result.stderr = ''
+        result.stdout = ('This is the test output for board %s, commit %s' %
+                (brd.target, commit.hash))
+        if ((boardnum >= 1 and boardnum >= commit.sequence) or
+                boardnum == 4 and commit.sequence == 6):
+            result.return_code = commit.return_code
+            result.stderr = (''.join(commit.error_list)
+                % {'basedir' : base_dir + '/.bm-work/00/'})
+        if stage == 'build':
+            target_dir = None
+            for arg in args:
+                if arg.startswith('O='):
+                    target_dir = arg[2:]
+
+            if not os.path.isdir(target_dir):
+                os.mkdir(target_dir)
+
+        result.combined = result.stdout + result.stderr
+        return result
+
+    def assertSummary(self, text, arch, plus, boards, ok=False):
+        col = self._col
+        expected_colour = col.GREEN if ok else col.RED
+        expect = '%10s: ' % arch
+        # TODO(sjg@chromium.org): If plus is '', we shouldn't need this
+        expect += ' ' + col.Color(expected_colour, plus)
+        expect += '  '
+        for board in boards:
+            expect += col.Color(expected_colour, ' %s' % board)
+        self.assertEqual(text, expect)
+
+    def testOutput(self):
+        """Test basic builder operation and output
+
+        This does a line-by-line verification of the summary output.
+        """
+        global base_dir
+
+        base_dir = tempfile.mkdtemp()
+        if not os.path.isdir(base_dir):
+            os.mkdir(base_dir)
+        build = builder.Builder(self.toolchains, base_dir, None, 1, 2,
+                                checkout=False, show_unknown=False)
+        build.do_make = self.Make
+        board_selected = self.boards.GetSelectedDict()
+
+        build.BuildBoards(self.commits, board_selected, keep_outputs=False,
+                          verbose=False)
+        lines = terminal.GetPrintTestLines()
+        count = 0
+        for line in lines:
+            if line.text.strip():
+                count += 1
+
+        # We should get one starting message, then an update for every commit
+        # built.
+        self.assertEqual(count, len(commits) * len(boards) + 1)
+        build.SetDisplayOptions(show_errors=True);
+        build.ShowSummary(self.commits, board_selected)
+        #terminal.EchoPrintTestLines()
+        lines = terminal.GetPrintTestLines()
+        self.assertEqual(lines[0].text, '01: %s' % commits[0][1])
+        self.assertEqual(lines[1].text, '02: %s' % commits[1][1])
+
+        # We expect all archs to fail
+        col = terminal.Color()
+        self.assertSummary(lines[2].text, 'sandbox', '+', ['board4'])
+        self.assertSummary(lines[3].text, 'arm', '+', ['board1'])
+        self.assertSummary(lines[4].text, 'powerpc', '+', ['board2', 'board3'])
+
+        # Now we should have the compiler warning
+        self.assertEqual(lines[5].text, 'w+%s' %
+                errors[0].rstrip().replace('\n', '\nw+'))
+        self.assertEqual(lines[5].colour, col.MAGENTA)
+
+        self.assertEqual(lines[6].text, '03: %s' % commits[2][1])
+        self.assertSummary(lines[7].text, 'sandbox', '+', ['board4'])
+        self.assertSummary(lines[8].text, 'arm', '', ['board1'], ok=True)
+        self.assertSummary(lines[9].text, 'powerpc', '+', ['board2', 'board3'])
+
+        # Compiler error
+        self.assertEqual(lines[10].text, '+%s' %
+                errors[1].rstrip().replace('\n', '\n+'))
+
+        self.assertEqual(lines[11].text, '04: %s' % commits[3][1])
+        self.assertSummary(lines[12].text, 'sandbox', '', ['board4'], ok=True)
+        self.assertSummary(lines[13].text, 'powerpc', '', ['board2', 'board3'],
+                ok=True)
+
+        # Compile error fixed
+        self.assertEqual(lines[14].text, '-%s' %
+                errors[1].rstrip().replace('\n', '\n-'))
+        self.assertEqual(lines[14].colour, col.GREEN)
+
+        self.assertEqual(lines[15].text, 'w+%s' %
+                errors[2].rstrip().replace('\n', '\nw+'))
+        self.assertEqual(lines[15].colour, col.MAGENTA)
+
+        self.assertEqual(lines[16].text, '05: %s' % commits[4][1])
+        self.assertSummary(lines[17].text, 'sandbox', '+', ['board4'])
+        self.assertSummary(lines[18].text, 'powerpc', '', ['board3'], ok=True)
+
+        # The second line of errors[3] is a duplicate, so buildman will drop it
+        expect = errors[3].rstrip().split('\n')
+        expect = [expect[0]] + expect[2:]
+        self.assertEqual(lines[19].text, '+%s' %
+                '\n'.join(expect).replace('\n', '\n+'))
+
+        self.assertEqual(lines[20].text, 'w-%s' %
+                errors[2].rstrip().replace('\n', '\nw-'))
+
+        self.assertEqual(lines[21].text, '06: %s' % commits[5][1])
+        self.assertSummary(lines[22].text, 'sandbox', '', ['board4'], ok=True)
+
+        # The second line of errors[3] is a duplicate, so buildman will drop it
+        expect = errors[3].rstrip().split('\n')
+        expect = [expect[0]] + expect[2:]
+        self.assertEqual(lines[23].text, '-%s' %
+                '\n'.join(expect).replace('\n', '\n-'))
+
+        self.assertEqual(lines[24].text, 'w-%s' %
+                errors[0].rstrip().replace('\n', '\nw-'))
+
+        self.assertEqual(lines[25].text, '07: %s' % commits[6][1])
+        self.assertSummary(lines[26].text, 'sandbox', '+', ['board4'])
+
+        # Pick out the correct error lines
+        expect_str = errors[4].rstrip().replace('%(basedir)s', '').split('\n')
+        expect = expect_str[3:8] + [expect_str[-1]]
+        self.assertEqual(lines[27].text, '+%s' %
+                '\n'.join(expect).replace('\n', '\n+'))
+
+        # Now the warnings lines
+        expect = [expect_str[0]] + expect_str[10:12] + [expect_str[9]]
+        self.assertEqual(lines[28].text, 'w+%s' %
+                '\n'.join(expect).replace('\n', '\nw+'))
+
+        self.assertEqual(len(lines), 29)
+        shutil.rmtree(base_dir)
+
+    def _testGit(self):
+        """Test basic builder operation by building a branch"""
+        base_dir = tempfile.mkdtemp()
+        if not os.path.isdir(base_dir):
+            os.mkdir(base_dir)
+        options = Options()
+        options.git = os.getcwd()
+        options.summary = False
+        options.jobs = None
+        options.dry_run = False
+        #options.git = os.path.join(base_dir, 'repo')
+        options.branch = 'test-buildman'
+        options.force_build = False
+        options.list_tool_chains = False
+        options.count = -1
+        options.git_dir = None
+        options.threads = None
+        options.show_unknown = False
+        options.quick = False
+        options.show_errors = False
+        options.keep_outputs = False
+        args = ['tegra20']
+        control.DoBuildman(options, args)
+        shutil.rmtree(base_dir)
+
+    def testBoardSingle(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards(['sandbox']),
+                         {'all': 1, 'sandbox': 1})
+
+    def testBoardArch(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards(['arm']),
+                         {'all': 2, 'arm': 2})
+
+    def testBoardArchSingle(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards(['arm sandbox']),
+                         {'all': 3, 'arm': 2, 'sandbox' : 1})
+
+    def testBoardArchSingleMultiWord(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards(['arm', 'sandbox']),
+                         {'all': 3, 'arm': 2, 'sandbox' : 1})
+
+    def testBoardSingleAnd(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards(['Tester & arm']),
+                         {'all': 2, 'Tester&arm': 2})
+
+    def testBoardTwoAnd(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards(['Tester', '&', 'arm',
+                                                   'Tester' '&', 'powerpc',
+                                                   'sandbox']),
+                         {'all': 5, 'Tester&powerpc': 2, 'Tester&arm': 2,
+                          'sandbox' : 1})
+
+    def testBoardAll(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards([]), {'all': 5})
+
+    def testBoardRegularExpression(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards(['T.*r&^Po']),
+                         {'T.*r&^Po': 2, 'all': 2})
+
+    def testBoardDuplicate(self):
+        """Test single board selection"""
+        self.assertEqual(self.boards.SelectBoards(['sandbox sandbox',
+                                                   'sandbox']),
+                         {'all': 1, 'sandbox': 1})
+    def CheckDirs(self, build, dirname):
+        self.assertEqual('base%s' % dirname, build._GetOutputDir(1))
+        self.assertEqual('base%s/fred' % dirname,
+                         build.GetBuildDir(1, 'fred'))
+        self.assertEqual('base%s/fred/done' % dirname,
+                         build.GetDoneFile(1, 'fred'))
+        self.assertEqual('base%s/fred/u-boot.sizes' % dirname,
+                         build.GetFuncSizesFile(1, 'fred', 'u-boot'))
+        self.assertEqual('base%s/fred/u-boot.objdump' % dirname,
+                         build.GetObjdumpFile(1, 'fred', 'u-boot'))
+        self.assertEqual('base%s/fred/err' % dirname,
+                         build.GetErrFile(1, 'fred'))
+
+    def testOutputDir(self):
+        build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2,
+                                checkout=False, show_unknown=False)
+        build.commits = self.commits
+        build.commit_count = len(self.commits)
+        subject = self.commits[1].subject.translate(builder.trans_valid_chars)
+        dirname ='/%02d_of_%02d_g%s_%s' % (2, build.commit_count, commits[1][0],
+                                           subject[:20])
+        self.CheckDirs(build, dirname)
+
+    def testOutputDirCurrent(self):
+        build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2,
+                                checkout=False, show_unknown=False)
+        build.commits = None
+        build.commit_count = 0
+        self.CheckDirs(build, '/current')
+
+    def testOutputDirNoSubdirs(self):
+        build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2,
+                                checkout=False, show_unknown=False,
+                                no_subdirs=True)
+        build.commits = None
+        build.commit_count = 0
+        self.CheckDirs(build, '')
+
+    def testToolchainAliases(self):
+        self.assertTrue(self.toolchains.Select('arm') != None)
+        with self.assertRaises(ValueError):
+            self.toolchains.Select('no-arch')
+        with self.assertRaises(ValueError):
+            self.toolchains.Select('x86')
+
+        self.toolchains = toolchain.Toolchains()
+        self.toolchains.Add('x86_64-linux-gcc', test=False)
+        self.assertTrue(self.toolchains.Select('x86') != None)
+
+        self.toolchains = toolchain.Toolchains()
+        self.toolchains.Add('i386-linux-gcc', test=False)
+        self.assertTrue(self.toolchains.Select('x86') != None)
+
+    def testToolchainDownload(self):
+        """Test that we can download toolchains"""
+        self.assertEqual('https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.9.0/x86_64-gcc-4.9.0-nolibc_arm-unknown-linux-gnueabi.tar.xz',
+            self.toolchains.LocateArchUrl('arm'))
+
+
+if __name__ == "__main__":
+    unittest.main()

u-boot/tools/buildman/toolchain.py

@@ -0,0 +1,544 @@
+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# SPDX-License-Identifier:	GPL-2.0+
+#
+
+import re
+import glob
+from HTMLParser import HTMLParser
+import os
+import sys
+import tempfile
+import urllib2
+
+import bsettings
+import command
+
+(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
+    PRIORITY_CALC) = range(4)
+
+# Simple class to collect links from a page
+class MyHTMLParser(HTMLParser):
+    def __init__(self, arch):
+        """Create a new parser
+
+        After the parser runs, self.links will be set to a list of the links
+        to .xz archives found in the page, and self.arch_link will be set to
+        the one for the given architecture (or None if not found).
+
+        Args:
+            arch: Architecture to search for
+        """
+        HTMLParser.__init__(self)
+        self.arch_link = None
+        self.links = []
+        self._match = '_%s-' % arch
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'a':
+            for tag, value in attrs:
+                if tag == 'href':
+                    if value and value.endswith('.xz'):
+                        self.links.append(value)
+                        if self._match in value:
+                            self.arch_link = value
+
+
+class Toolchain:
+    """A single toolchain
+
+    Public members:
+        gcc: Full path to C compiler
+        path: Directory path containing C compiler
+        cross: Cross compile string, e.g. 'arm-linux-'
+        arch: Architecture of toolchain as determined from the first
+                component of the filename. E.g. arm-linux-gcc becomes arm
+        priority: Toolchain priority (0=highest, 20=lowest)
+    """
+    def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
+                 arch=None):
+        """Create a new toolchain object.
+
+        Args:
+            fname: Filename of the gcc component
+            test: True to run the toolchain to test it
+            verbose: True to print out the information
+            priority: Priority to use for this toolchain, or PRIORITY_CALC to
+                calculate it
+        """
+        self.gcc = fname
+        self.path = os.path.dirname(fname)
+
+        # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
+        # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
+        basename = os.path.basename(fname)
+        pos = basename.rfind('-')
+        self.cross = basename[:pos + 1] if pos != -1 else ''
+
+        # The architecture is the first part of the name
+        pos = self.cross.find('-')
+        if arch:
+            self.arch = arch
+        else:
+            self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
+
+        env = self.MakeEnvironment(False)
+
+        # As a basic sanity check, run the C compiler with --version
+        cmd = [fname, '--version']
+        if priority == PRIORITY_CALC:
+            self.priority = self.GetPriority(fname)
+        else:
+            self.priority = priority
+        if test:
+            result = command.RunPipe([cmd], capture=True, env=env,
+                                     raise_on_error=False)
+            self.ok = result.return_code == 0
+            if verbose:
+                print 'Tool chain test: ',
+                if self.ok:
+                    print "OK, arch='%s', priority %d" % (self.arch,
+                                                          self.priority)
+                else:
+                    print 'BAD'
+                    print 'Command: ', cmd
+                    print result.stdout
+                    print result.stderr
+        else:
+            self.ok = True
+
+    def GetPriority(self, fname):
+        """Return the priority of the toolchain.
+
+        Toolchains are ranked according to their suitability by their
+        filename prefix.
+
+        Args:
+            fname: Filename of toolchain
+        Returns:
+            Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
+        """
+        priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
+            '-none-linux-gnueabi', '-uclinux', '-none-eabi',
+            '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
+        for prio in range(len(priority_list)):
+            if priority_list[prio] in fname:
+                return PRIORITY_CALC + prio
+        return PRIORITY_CALC + prio
+
+    def MakeEnvironment(self, full_path):
+        """Returns an environment for using the toolchain.
+
+        Thie takes the current environment and adds CROSS_COMPILE so that
+        the tool chain will operate correctly.
+
+        Args:
+            full_path: Return the full path in CROSS_COMPILE and don't set
+                PATH
+        """
+        env = dict(os.environ)
+        if full_path:
+            env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
+        else:
+            env['CROSS_COMPILE'] = self.cross
+            env['PATH'] = self.path + ':' + env['PATH']
+
+        return env
+
+
+class Toolchains:
+    """Manage a list of toolchains for building U-Boot
+
+    We select one toolchain for each architecture type
+
+    Public members:
+        toolchains: Dict of Toolchain objects, keyed by architecture name
+        prefixes: Dict of prefixes to check, keyed by architecture. This can
+            be a full path and toolchain prefix, for example
+            {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
+            something on the search path, for example
+            {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
+        paths: List of paths to check for toolchains (may contain wildcards)
+    """
+
+    def __init__(self):
+        self.toolchains = {}
+        self.prefixes = {}
+        self.paths = []
+        self._make_flags = dict(bsettings.GetItems('make-flags'))
+
+    def GetPathList(self):
+        """Get a list of available toolchain paths
+
+        Returns:
+            List of strings, each a path to a toolchain mentioned in the
+            [toolchain] section of the settings file.
+        """
+        toolchains = bsettings.GetItems('toolchain')
+        if not toolchains:
+            print ('Warning: No tool chains - please add a [toolchain] section'
+                 ' to your buildman config file %s. See README for details' %
+                 bsettings.config_fname)
+
+        paths = []
+        for name, value in toolchains:
+            if '*' in value:
+                paths += glob.glob(value)
+            else:
+                paths.append(value)
+        return paths
+
+    def GetSettings(self):
+      self.prefixes = bsettings.GetItems('toolchain-prefix')
+      self.paths += self.GetPathList()
+
+    def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
+            arch=None):
+        """Add a toolchain to our list
+
+        We select the given toolchain as our preferred one for its
+        architecture if it is a higher priority than the others.
+
+        Args:
+            fname: Filename of toolchain's gcc driver
+            test: True to run the toolchain to test it
+            priority: Priority to use for this toolchain
+            arch: Toolchain architecture, or None if not known
+        """
+        toolchain = Toolchain(fname, test, verbose, priority, arch)
+        add_it = toolchain.ok
+        if toolchain.arch in self.toolchains:
+            add_it = (toolchain.priority <
+                        self.toolchains[toolchain.arch].priority)
+        if add_it:
+            self.toolchains[toolchain.arch] = toolchain
+        elif verbose:
+            print ("Toolchain '%s' at priority %d will be ignored because "
+                   "another toolchain for arch '%s' has priority %d" %
+                   (toolchain.gcc, toolchain.priority, toolchain.arch,
+                    self.toolchains[toolchain.arch].priority))
+
+    def ScanPath(self, path, verbose):
+        """Scan a path for a valid toolchain
+
+        Args:
+            path: Path to scan
+            verbose: True to print out progress information
+        Returns:
+            Filename of C compiler if found, else None
+        """
+        fnames = []
+        for subdir in ['.', 'bin', 'usr/bin']:
+            dirname = os.path.join(path, subdir)
+            if verbose: print "      - looking in '%s'" % dirname
+            for fname in glob.glob(dirname + '/*gcc'):
+                if verbose: print "         - found '%s'" % fname
+                fnames.append(fname)
+        return fnames
+
+    def ScanPathEnv(self, fname):
+        """Scan the PATH environment variable for a given filename.
+
+        Args:
+            fname: Filename to scan for
+        Returns:
+            List of matching pathanames, or [] if none
+        """
+        pathname_list = []
+        for path in os.environ["PATH"].split(os.pathsep):
+            path = path.strip('"')
+            pathname = os.path.join(path, fname)
+            if os.path.exists(pathname):
+                pathname_list.append(pathname)
+        return pathname_list
+
+    def Scan(self, verbose):
+        """Scan for available toolchains and select the best for each arch.
+
+        We look for all the toolchains we can file, figure out the
+        architecture for each, and whether it works. Then we select the
+        highest priority toolchain for each arch.
+
+        Args:
+            verbose: True to print out progress information
+        """
+        if verbose: print 'Scanning for tool chains'
+        for name, value in self.prefixes:
+            if verbose: print "   - scanning prefix '%s'" % value
+            if os.path.exists(value):
+                self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
+                continue
+            fname = value + 'gcc'
+            if os.path.exists(fname):
+                self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
+                continue
+            fname_list = self.ScanPathEnv(fname)
+            for f in fname_list:
+                self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
+            if not fname_list:
+                raise ValueError, ("No tool chain found for prefix '%s'" %
+                                   value)
+        for path in self.paths:
+            if verbose: print "   - scanning path '%s'" % path
+            fnames = self.ScanPath(path, verbose)
+            for fname in fnames:
+                self.Add(fname, True, verbose)
+
+    def List(self):
+        """List out the selected toolchains for each architecture"""
+        print 'List of available toolchains (%d):' % len(self.toolchains)
+        if len(self.toolchains):
+            for key, value in sorted(self.toolchains.iteritems()):
+                print '%-10s: %s' % (key, value.gcc)
+        else:
+            print 'None'
+
+    def Select(self, arch):
+        """Returns the toolchain for a given architecture
+
+        Args:
+            args: Name of architecture (e.g. 'arm', 'ppc_8xx')
+
+        returns:
+            toolchain object, or None if none found
+        """
+        for tag, value in bsettings.GetItems('toolchain-alias'):
+            if arch == tag:
+                for alias in value.split():
+                    if alias in self.toolchains:
+                        return self.toolchains[alias]
+
+        if not arch in self.toolchains:
+            raise ValueError, ("No tool chain found for arch '%s'" % arch)
+        return self.toolchains[arch]
+
+    def ResolveReferences(self, var_dict, args):
+        """Resolve variable references in a string
+
+        This converts ${blah} within the string to the value of blah.
+        This function works recursively.
+
+        Args:
+            var_dict: Dictionary containing variables and their values
+            args: String containing make arguments
+        Returns:
+            Resolved string
+
+        >>> bsettings.Setup()
+        >>> tcs = Toolchains()
+        >>> tcs.Add('fred', False)
+        >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
+                        'second' : '2nd'}
+        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
+        'this=OBLIQUE_set'
+        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
+        'this=OBLIQUE_setfi2ndrstnd'
+        """
+        re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
+
+        while True:
+            m = re_var.search(args)
+            if not m:
+                break
+            lookup = m.group(0)[2:-1]
+            value = var_dict.get(lookup, '')
+            args = args[:m.start(0)] + value + args[m.end(0):]
+        return args
+
+    def GetMakeArguments(self, board):
+        """Returns 'make' arguments for a given board
+
+        The flags are in a section called 'make-flags'. Flags are named
+        after the target they represent, for example snapper9260=TESTING=1
+        will pass TESTING=1 to make when building the snapper9260 board.
+
+        References to other boards can be added in the string also. For
+        example:
+
+        [make-flags]
+        at91-boards=ENABLE_AT91_TEST=1
+        snapper9260=${at91-boards} BUILD_TAG=442
+        snapper9g45=${at91-boards} BUILD_TAG=443
+
+        This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
+        and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
+
+        A special 'target' variable is set to the board target.
+
+        Args:
+            board: Board object for the board to check.
+        Returns:
+            'make' flags for that board, or '' if none
+        """
+        self._make_flags['target'] = board.target
+        arg_str = self.ResolveReferences(self._make_flags,
+                           self._make_flags.get(board.target, ''))
+        args = arg_str.split(' ')
+        i = 0
+        while i < len(args):
+            if not args[i]:
+                del args[i]
+            else:
+                i += 1
+        return args
+
+    def LocateArchUrl(self, fetch_arch):
+        """Find a toolchain available online
+
+        Look in standard places for available toolchains. At present the
+        only standard place is at kernel.org.
+
+        Args:
+            arch: Architecture to look for, or 'list' for all
+        Returns:
+            If fetch_arch is 'list', a tuple:
+                Machine architecture (e.g. x86_64)
+                List of toolchains
+            else
+                URL containing this toolchain, if avaialble, else None
+        """
+        arch = command.OutputOneLine('uname', '-m')
+        base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
+        versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
+        links = []
+        for version in versions:
+            url = '%s/%s/%s/' % (base, arch, version)
+            print 'Checking: %s' % url
+            response = urllib2.urlopen(url)
+            html = response.read()
+            parser = MyHTMLParser(fetch_arch)
+            parser.feed(html)
+            if fetch_arch == 'list':
+                links += parser.links
+            elif parser.arch_link:
+                return url + parser.arch_link
+        if fetch_arch == 'list':
+            return arch, links
+        return None
+
+    def Download(self, url):
+        """Download a file to a temporary directory
+
+        Args:
+            url: URL to download
+        Returns:
+            Tuple:
+                Temporary directory name
+                Full path to the downloaded archive file in that directory,
+                    or None if there was an error while downloading
+        """
+        print 'Downloading: %s' % url
+        leaf = url.split('/')[-1]
+        tmpdir = tempfile.mkdtemp('.buildman')
+        response = urllib2.urlopen(url)
+        fname = os.path.join(tmpdir, leaf)
+        fd = open(fname, 'wb')
+        meta = response.info()
+        size = int(meta.getheaders('Content-Length')[0])
+        done = 0
+        block_size = 1 << 16
+        status = ''
+
+        # Read the file in chunks and show progress as we go
+        while True:
+            buffer = response.read(block_size)
+            if not buffer:
+                print chr(8) * (len(status) + 1), '\r',
+                break
+
+            done += len(buffer)
+            fd.write(buffer)
+            status = r'%10d MiB  [%3d%%]' % (done / 1024 / 1024,
+                                             done * 100 / size)
+            status = status + chr(8) * (len(status) + 1)
+            print status,
+            sys.stdout.flush()
+        fd.close()
+        if done != size:
+            print 'Error, failed to download'
+            os.remove(fname)
+            fname = None
+        return tmpdir, fname
+
+    def Unpack(self, fname, dest):
+        """Unpack a tar file
+
+        Args:
+            fname: Filename to unpack
+            dest: Destination directory
+        Returns:
+            Directory name of the first entry in the archive, without the
+            trailing /
+        """
+        stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
+        return stdout.splitlines()[0][:-1]
+
+    def TestSettingsHasPath(self, path):
+        """Check if builmand will find this toolchain
+
+        Returns:
+            True if the path is in settings, False if not
+        """
+        paths = self.GetPathList()
+        return path in paths
+
+    def ListArchs(self):
+        """List architectures with available toolchains to download"""
+        host_arch, archives = self.LocateArchUrl('list')
+        re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
+        arch_set = set()
+        for archive in archives:
+            # Remove the host architecture from the start
+            arch = re_arch.match(archive[len(host_arch):])
+            if arch:
+                arch_set.add(arch.group(1))
+        return sorted(arch_set)
+
+    def FetchAndInstall(self, arch):
+        """Fetch and install a new toolchain
+
+        arch:
+            Architecture to fetch, or 'list' to list
+        """
+        # Fist get the URL for this architecture
+        url = self.LocateArchUrl(arch)
+        if not url:
+            print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
+                   arch)
+            return 2
+        home = os.environ['HOME']
+        dest = os.path.join(home, '.buildman-toolchains')
+        if not os.path.exists(dest):
+            os.mkdir(dest)
+
+        # Download the tar file for this toolchain and unpack it
+        tmpdir, tarfile = self.Download(url)
+        if not tarfile:
+            return 1
+        print 'Unpacking to: %s' % dest,
+        sys.stdout.flush()
+        path = self.Unpack(tarfile, dest)
+        os.remove(tarfile)
+        os.rmdir(tmpdir)
+        print
+
+        # Check that the toolchain works
+        print 'Testing'
+        dirpath = os.path.join(dest, path)
+        compiler_fname_list = self.ScanPath(dirpath, True)
+        if not compiler_fname_list:
+            print 'Could not locate C compiler - fetch failed.'
+            return 1
+        if len(compiler_fname_list) != 1:
+            print ('Internal error, ambiguous toolchains: %s' %
+                   (', '.join(compiler_fname)))
+            return 1
+        toolchain = Toolchain(compiler_fname_list[0], True, True)
+
+        # Make sure that it will be found by buildman
+        if not self.TestSettingsHasPath(dirpath):
+            print ("Adding 'download' to config file '%s'" %
+                   bsettings.config_fname)
+            tools_dir = os.path.dirname(dirpath)
+            bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)
+        return 0