1 # Copyright (c) 2013 The Chromium OS Authors.
3 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5 # SPDX-License-Identifier: GPL-2.0+
10 from datetime import datetime, timedelta
30 Please see README for user documentation, and you should be familiar with
31 that before trying to make sense of this.
33 Buildman works by keeping the machine as busy as possible, building different
34 commits for different boards on multiple CPUs at once.
36 The source repo (self.git_dir) contains all the commits to be built. Each
37 thread works on a single board at a time. It checks out the first commit,
38 configures it for that board, then builds it. Then it checks out the next
39 commit and builds it (typically without re-configuring). When it runs out
40 of commits, it gets another job from the builder and starts again with that
43 Clearly the builder threads could work either way - they could check out a
44 commit and then built it for all boards. Using separate directories for each
45 commit/board pair they could leave their build product around afterwards
48 The intent behind building a single board for multiple commits, is to make
49 use of incremental builds. Since each commit is built incrementally from
50 the previous one, builds are faster. Reconfiguring for a different board
51 removes all intermediate object files.
53 Many threads can be working at once, but each has its own working directory.
54 When a thread finishes a build, it puts the output files into a result
57 The base directory used by buildman is normally '../<branch>', i.e.
58 a directory higher than the source repository and named after the branch
61 Within the base directory, we have one subdirectory for each commit. Within
62 that is one subdirectory for each board. Within that is the build output for
63 that commit/board combination.
65 Buildman also create working directories for each thread, in a .bm-work/
66 subdirectory in the base dir.
68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
69 'seaboard', and say that us-net has two commits. We will have directories
72 us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
84 00/ working directory for thread 0 (contains source checkout)
86 01/ working directory for thread 1
89 u-boot/ source directory
93 # Possible build outcomes
94 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
96 # Translate a commit subject into a valid filename
97 trans_valid_chars = string.maketrans("/: ", "---")
101 """Make a directory if it doesn't already exist.
104 dirname: Directory to create
108 except OSError as err:
109 if err.errno == errno.EEXIST:
115 """Holds information about a job to be performed by a thread
118 board: Board object to build
119 commits: List of commit options to build.
126 class ResultThread(threading.Thread):
127 """This thread processes results from builder threads.
129 It simply passes the results on to the builder. There is only one
130 result thread, and this helps to serialise the build output.
132 def __init__(self, builder):
133 """Set up a new result thread
136 builder: Builder which will be sent each result
138 threading.Thread.__init__(self)
139 self.builder = builder
142 """Called to start up the result thread.
144 We collect the next result job and pass it on to the build.
147 result = self.builder.out_queue.get()
148 self.builder.ProcessResult(result)
149 self.builder.out_queue.task_done()
152 class BuilderThread(threading.Thread):
153 """This thread builds U-Boot for a particular board.
155 An input queue provides each new job. We run 'make' to build U-Boot
156 and then pass the results on to the output queue.
159 builder: The builder which contains information we might need
160 thread_num: Our thread number (0-n-1), used to decide on a
163 def __init__(self, builder, thread_num):
164 """Set up a new builder thread"""
165 threading.Thread.__init__(self)
166 self.builder = builder
167 self.thread_num = thread_num
169 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
170 """Run 'make' on a particular commit and board.
172 The source code will already be checked out, so the 'commit'
173 argument is only for information.
176 commit: Commit object that is being built
177 brd: Board object that is being built
178 stage: Stage of the build. Valid stages are:
179 distclean - can be called to clean source
180 config - called to configure for a board
181 build - the main make invocation - it does the build
182 args: A list of arguments to pass to 'make'
183 kwargs: A list of keyword arguments to pass to command.RunPipe()
188 return self.builder.do_make(commit, brd, stage, cwd, *args,
191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
192 force_build_failures):
193 """Build a particular commit.
195 If the build is already done, and we are not forcing a build, we skip
196 the build and just return the previously-saved results.
199 commit_upto: Commit number to build (0...n-1)
200 brd: Board object to build
201 work_dir: Directory to which the source will be checked out
202 do_config: True to run a make <board>_defconfig on the source
203 force_build: Force a build even if one was previously done
204 force_build_failures: Force a bulid if the previous result showed
209 - CommandResult object containing the results of the build
210 - boolean indicating whether 'make config' is still needed
212 # Create a default result - it will be overwritte by the call to
213 # self.Make() below, in the event that we do a build.
214 result = command.CommandResult()
215 result.return_code = 0
216 if self.builder.in_tree:
219 out_dir = os.path.join(work_dir, 'build')
221 # Check if the job was already completed last time
222 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
223 result.already_done = os.path.exists(done_file)
224 will_build = (force_build or force_build_failures or
225 not result.already_done)
226 if result.already_done and will_build:
227 # Get the return code from that build and use it
228 with open(done_file, 'r') as fd:
229 result.return_code = int(fd.readline())
230 err_file = self.builder.GetErrFile(commit_upto, brd.target)
231 if os.path.exists(err_file) and os.stat(err_file).st_size:
232 result.stderr = 'bad'
233 elif not force_build:
234 # The build passed, so no need to build it again
238 # We are going to have to build it. First, get a toolchain
239 if not self.toolchain:
241 self.toolchain = self.builder.toolchains.Select(brd.arch)
242 except ValueError as err:
243 result.return_code = 10
245 result.stderr = str(err)
246 # TODO(sjg@chromium.org): This gets swallowed, but needs
250 # Checkout the right commit
251 if self.builder.commits:
252 commit = self.builder.commits[commit_upto]
253 if self.builder.checkout:
254 git_dir = os.path.join(work_dir, '.git')
255 gitutil.Checkout(commit.hash, git_dir, work_dir,
260 # Set up the environment and command line
261 env = self.toolchain.MakeEnvironment()
265 if not self.builder.in_tree:
266 if commit_upto is None:
267 # In this case we are building in the original source
268 # directory (i.e. the current directory where buildman
269 # is invoked. The output directory is set to this
270 # thread's selected work directory.
272 # Symlinks can confuse U-Boot's Makefile since
273 # we may use '..' in our path, so remove them.
274 work_dir = os.path.realpath(work_dir)
275 args.append('O=%s/build' % work_dir)
278 args.append('O=build')
280 if self.builder.num_jobs is not None:
281 args.extend(['-j', str(self.builder.num_jobs)])
282 config_args = ['%s_defconfig' % brd.target]
284 args.extend(self.builder.toolchains.GetMakeArguments(brd))
286 # If we need to reconfigure, do that now
288 result = self.Make(commit, brd, 'distclean', cwd,
289 'distclean', *args, env=env)
290 result = self.Make(commit, brd, 'config', cwd,
291 *(args + config_args), env=env)
292 config_out = result.combined
293 do_config = False # No need to configure next time
294 if result.return_code == 0:
295 result = self.Make(commit, brd, 'build', cwd, *args,
297 result.stdout = config_out + result.stdout
299 result.return_code = 1
300 result.stderr = 'No tool chain for %s\n' % brd.arch
301 result.already_done = False
303 result.toolchain = self.toolchain
305 result.commit_upto = commit_upto
306 result.out_dir = out_dir
307 return result, do_config
309 def _WriteResult(self, result, keep_outputs):
310 """Write a built result to the output directory.
313 result: CommandResult object containing result to write
314 keep_outputs: True to store the output binaries, False
318 if result.return_code < 0:
322 if result.stderr and 'No child processes' in result.stderr:
325 if result.already_done:
328 # Write the output and stderr
329 output_dir = self.builder._GetOutputDir(result.commit_upto)
331 build_dir = self.builder.GetBuildDir(result.commit_upto,
335 outfile = os.path.join(build_dir, 'log')
336 with open(outfile, 'w') as fd:
338 fd.write(result.stdout)
340 errfile = self.builder.GetErrFile(result.commit_upto,
343 with open(errfile, 'w') as fd:
344 fd.write(result.stderr)
345 elif os.path.exists(errfile):
349 # Write the build result and toolchain information.
350 done_file = self.builder.GetDoneFile(result.commit_upto,
352 with open(done_file, 'w') as fd:
353 fd.write('%s' % result.return_code)
354 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
355 print >>fd, 'gcc', result.toolchain.gcc
356 print >>fd, 'path', result.toolchain.path
357 print >>fd, 'cross', result.toolchain.cross
358 print >>fd, 'arch', result.toolchain.arch
359 fd.write('%s' % result.return_code)
361 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
362 print >>fd, 'gcc', result.toolchain.gcc
363 print >>fd, 'path', result.toolchain.path
365 # Write out the image and function size information and an objdump
366 env = result.toolchain.MakeEnvironment()
368 for fname in ['u-boot', 'spl/u-boot-spl']:
369 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
370 nm_result = command.RunPipe([cmd], capture=True,
371 capture_stderr=True, cwd=result.out_dir,
372 raise_on_error=False, env=env)
374 nm = self.builder.GetFuncSizesFile(result.commit_upto,
375 result.brd.target, fname)
376 with open(nm, 'w') as fd:
377 print >>fd, nm_result.stdout,
379 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
380 dump_result = command.RunPipe([cmd], capture=True,
381 capture_stderr=True, cwd=result.out_dir,
382 raise_on_error=False, env=env)
384 if dump_result.stdout:
385 objdump = self.builder.GetObjdumpFile(result.commit_upto,
386 result.brd.target, fname)
387 with open(objdump, 'w') as fd:
388 print >>fd, dump_result.stdout,
389 for line in dump_result.stdout.splitlines():
390 fields = line.split()
391 if len(fields) > 5 and fields[1] == '.rodata':
392 rodata_size = fields[2]
394 cmd = ['%ssize' % self.toolchain.cross, fname]
395 size_result = command.RunPipe([cmd], capture=True,
396 capture_stderr=True, cwd=result.out_dir,
397 raise_on_error=False, env=env)
398 if size_result.stdout:
399 lines.append(size_result.stdout.splitlines()[1] + ' ' +
402 # Write out the image sizes file. This is similar to the output
403 # of binutil's 'size' utility, but it omits the header line and
404 # adds an additional hex value at the end of each line for the
407 sizes = self.builder.GetSizesFile(result.commit_upto,
409 with open(sizes, 'w') as fd:
410 print >>fd, '\n'.join(lines)
412 # Now write the actual build output
414 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
415 'include/autoconf.mk', 'spl/u-boot-spl',
416 'spl/u-boot-spl.bin']
417 for pattern in patterns:
418 file_list = glob.glob(os.path.join(result.out_dir, pattern))
419 for fname in file_list:
420 shutil.copy(fname, build_dir)
423 def RunJob(self, job):
426 A job consists of a building a list of commits for a particular board.
432 work_dir = self.builder.GetThreadDir(self.thread_num)
433 self.toolchain = None
435 # Run 'make board_defconfig' on the first commit
439 for commit_upto in range(0, len(job.commits), job.step):
440 result, request_config = self.RunCommit(commit_upto, brd,
442 force_build or self.builder.force_build,
443 self.builder.force_build_failures)
444 failed = result.return_code or result.stderr
445 did_config = do_config
446 if failed and not do_config:
447 # If our incremental build failed, try building again
449 if self.builder.force_config_on_failure:
450 result, request_config = self.RunCommit(commit_upto,
451 brd, work_dir, True, True, False)
453 if not self.builder.force_reconfig:
454 do_config = request_config
456 # If we built that commit, then config is done. But if we got
457 # an warning, reconfig next time to force it to build the same
458 # files that created warnings this time. Otherwise an
459 # incremental build may not build the same file, and we will
460 # think that the warning has gone away.
461 # We could avoid this by using -Werror everywhere...
462 # For errors, the problem doesn't happen, since presumably
463 # the build stopped and didn't generate output, so will retry
464 # that file next time. So we could detect warnings and deal
465 # with them specially here. For now, we just reconfigure if
466 # anything goes work.
467 # Of course this is substantially slower if there are build
468 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
470 if (failed and not result.already_done and not did_config and
471 self.builder.force_config_on_failure):
472 # If this build failed, try the next one with a
474 # Sometimes if the board_config.h file changes it can mess
475 # with dependencies, and we get:
476 # make: *** No rule to make target `include/autoconf.mk',
477 # needed by `depend'.
482 if self.builder.force_config_on_failure:
485 result.commit_upto = commit_upto
486 if result.return_code < 0:
487 raise ValueError('Interrupt')
489 # We have the build results, so output the result
490 self._WriteResult(result, job.keep_outputs)
491 self.builder.out_queue.put(result)
493 # Just build the currently checked-out build
494 result, request_config = self.RunCommit(None, brd, work_dir, True,
495 True, self.builder.force_build_failures)
496 result.commit_upto = 0
497 self._WriteResult(result, job.keep_outputs)
498 self.builder.out_queue.put(result)
501 """Our thread's run function
503 This thread picks a job from the queue, runs it, and then goes to the
508 job = self.builder.queue.get()
509 if self.builder.active and alive:
513 if self.builder.active and alive:
515 except Exception as err:
519 self.builder.queue.task_done()
523 """Class for building U-Boot for a particular commit.
525 Public members: (many should ->private)
526 active: True if the builder is active and has not been stopped
527 already_done: Number of builds already completed
528 base_dir: Base directory to use for builder
529 checkout: True to check out source, False to skip that step.
530 This is used for testing.
531 col: terminal.Color() object
532 count: Number of commits to build
533 do_make: Method to call to invoke Make
534 fail: Number of builds that failed due to error
535 force_build: Force building even if a build already exists
536 force_config_on_failure: If a commit fails for a board, disable
537 incremental building for the next commit we build for that
538 board, so that we will see all warnings/errors again.
539 force_build_failures: If a previously-built build (i.e. built on
540 a previous run of buildman) is marked as failed, rebuild it.
541 git_dir: Git directory containing source repository
542 last_line_len: Length of the last line we printed (used for erasing
543 it with new progress information)
544 num_jobs: Number of jobs to run at once (passed to make as -j)
545 num_threads: Number of builder threads to run
546 out_queue: Queue of results to process
547 re_make_err: Compiled regular expression for ignore_lines
548 queue: Queue of jobs to run
549 threads: List of active threads
550 toolchains: Toolchains object to use for building
551 upto: Current commit number we are building (0.count-1)
552 warned: Number of builds that produced at least one warning
553 force_reconfig: Reconfigure U-Boot on each comiit. This disables
554 incremental building, where buildman reconfigures on the first
555 commit for a baord, and then just does an incremental build for
556 the following commits. In fact buildman will reconfigure and
557 retry for any failing commits, so generally the only effect of
558 this option is to slow things down.
559 in_tree: Build U-Boot in-tree instead of specifying an output
560 directory separate from the source code. This option is really
561 only useful for testing in-tree builds.
564 _base_board_dict: Last-summarised Dict of boards
565 _base_err_lines: Last-summarised list of errors
566 _build_period_us: Time taken for a single build (float object).
567 _complete_delay: Expected delay until completion (timedelta)
568 _next_delay_update: Next time we plan to display a progress update
570 _show_unknown: Show unknown boards (those not built) in summary
571 _timestamps: List of timestamps for the completion of the last
572 last _timestamp_count builds. Each is a datetime object.
573 _timestamp_count: Number of timestamps to keep in our list.
574 _working_dir: Base working directory containing all threads
577 """Records a build outcome for a single make invocation
580 rc: Outcome value (OUTCOME_...)
581 err_lines: List of error lines or [] if none
582 sizes: Dictionary of image size information, keyed by filename
583 - Each value is itself a dictionary containing
584 values for 'text', 'data' and 'bss', being the integer
585 size in bytes of each section.
586 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
587 value is itself a dictionary:
589 value: Size of function in bytes
591 def __init__(self, rc, err_lines, sizes, func_sizes):
593 self.err_lines = err_lines
595 self.func_sizes = func_sizes
597 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
598 gnu_make='make', checkout=True, show_unknown=True, step=1):
599 """Create a new Builder object
602 toolchains: Toolchains object to use for building
603 base_dir: Base directory to use for builder
604 git_dir: Git directory containing source repository
605 num_threads: Number of builder threads to run
606 num_jobs: Number of jobs to run at once (passed to make as -j)
607 gnu_make: the command name of GNU Make.
608 checkout: True to check out source, False to skip that step.
609 This is used for testing.
610 show_unknown: Show unknown boards (those not built) in summary
611 step: 1 to process every commit, n to process every nth commit
613 self.toolchains = toolchains
614 self.base_dir = base_dir
615 self._working_dir = os.path.join(base_dir, '.bm-work')
618 self.do_make = self.Make
619 self.gnu_make = gnu_make
620 self.checkout = checkout
621 self.num_threads = num_threads
622 self.num_jobs = num_jobs
623 self.already_done = 0
624 self.force_build = False
625 self.git_dir = git_dir
626 self._show_unknown = show_unknown
627 self._timestamp_count = 10
628 self._build_period_us = None
629 self._complete_delay = None
630 self._next_delay_update = datetime.now()
631 self.force_config_on_failure = True
632 self.force_build_failures = False
633 self.force_reconfig = False
637 self.col = terminal.Color()
639 self.queue = Queue.Queue()
640 self.out_queue = Queue.Queue()
641 for i in range(self.num_threads):
642 t = BuilderThread(self, i)
645 self.threads.append(t)
647 self.last_line_len = 0
648 t = ResultThread(self)
651 self.threads.append(t)
653 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
654 self.re_make_err = re.compile('|'.join(ignore_lines))
657 """Get rid of all threads created by the builder"""
658 for t in self.threads:
661 def _AddTimestamp(self):
662 """Add a new timestamp to the list and record the build period.
664 The build period is the length of time taken to perform a single
665 build (one board, one commit).
668 self._timestamps.append(now)
669 count = len(self._timestamps)
670 delta = self._timestamps[-1] - self._timestamps[0]
671 seconds = delta.total_seconds()
673 # If we have enough data, estimate build period (time taken for a
674 # single build) and therefore completion time.
675 if count > 1 and self._next_delay_update < now:
676 self._next_delay_update = now + timedelta(seconds=2)
678 self._build_period = float(seconds) / count
679 todo = self.count - self.upto
680 self._complete_delay = timedelta(microseconds=
681 self._build_period * todo * 1000000)
683 self._complete_delay -= timedelta(
684 microseconds=self._complete_delay.microseconds)
687 self._timestamps.popleft()
690 def ClearLine(self, length):
691 """Clear any characters on the current line
693 Make way for a new line of length 'length', by outputting enough
694 spaces to clear out the old line. Then remember the new length for
698 length: Length of new line, in characters
700 if length < self.last_line_len:
701 print ' ' * (self.last_line_len - length),
703 self.last_line_len = length
706 def SelectCommit(self, commit, checkout=True):
707 """Checkout the selected commit for this build
710 if checkout and self.checkout:
711 gitutil.Checkout(commit.hash)
713 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
717 commit: Commit object that is being built
718 brd: Board object that is being built
719 stage: Stage that we are at (distclean, config, build)
720 cwd: Directory where make should be run
721 args: Arguments to pass to make
722 kwargs: Arguments to pass to command.RunPipe()
724 cmd = [self.gnu_make] + list(args)
725 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
726 cwd=cwd, raise_on_error=False, **kwargs)
729 def ProcessResult(self, result):
730 """Process the result of a build, showing progress information
733 result: A CommandResult object
735 col = terminal.Color()
737 target = result.brd.target
739 if result.return_code < 0:
745 if result.return_code != 0:
749 if result.already_done:
750 self.already_done += 1
752 target = '(starting)'
754 # Display separate counts for ok, warned and fail
755 ok = self.upto - self.warned - self.fail
756 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
757 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
758 line += self.col.Color(self.col.RED, '%5d' % self.fail)
760 name = ' /%-5d ' % self.count
762 # Add our current completion time estimate
764 if self._complete_delay:
765 name += '%s : ' % self._complete_delay
766 # When building all boards for a commit, we can print a commit
768 if result and result.commit_upto is None:
769 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
774 length = 13 + len(name)
775 self.ClearLine(length)
777 def _GetOutputDir(self, commit_upto):
778 """Get the name of the output directory for a commit number
780 The output directory is typically .../<branch>/<commit>.
783 commit_upto: Commit number to use (0..self.count-1)
786 commit = self.commits[commit_upto]
787 subject = commit.subject.translate(trans_valid_chars)
788 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
789 self.commit_count, commit.hash, subject[:20]))
791 commit_dir = 'current'
792 output_dir = os.path.join(self.base_dir, commit_dir)
795 def GetBuildDir(self, commit_upto, target):
796 """Get the name of the build directory for a commit number
798 The build directory is typically .../<branch>/<commit>/<target>.
801 commit_upto: Commit number to use (0..self.count-1)
804 output_dir = self._GetOutputDir(commit_upto)
805 return os.path.join(output_dir, target)
807 def GetDoneFile(self, commit_upto, target):
808 """Get the name of the done file for a commit number
811 commit_upto: Commit number to use (0..self.count-1)
814 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
816 def GetSizesFile(self, commit_upto, target):
817 """Get the name of the sizes file for a commit number
820 commit_upto: Commit number to use (0..self.count-1)
823 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
825 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
826 """Get the name of the funcsizes file for a commit number and ELF file
829 commit_upto: Commit number to use (0..self.count-1)
831 elf_fname: Filename of elf image
833 return os.path.join(self.GetBuildDir(commit_upto, target),
834 '%s.sizes' % elf_fname.replace('/', '-'))
836 def GetObjdumpFile(self, commit_upto, target, elf_fname):
837 """Get the name of the objdump file for a commit number and ELF file
840 commit_upto: Commit number to use (0..self.count-1)
842 elf_fname: Filename of elf image
844 return os.path.join(self.GetBuildDir(commit_upto, target),
845 '%s.objdump' % elf_fname.replace('/', '-'))
847 def GetErrFile(self, commit_upto, target):
848 """Get the name of the err file for a commit number
851 commit_upto: Commit number to use (0..self.count-1)
854 output_dir = self.GetBuildDir(commit_upto, target)
855 return os.path.join(output_dir, 'err')
857 def FilterErrors(self, lines):
858 """Filter out errors in which we have no interest
860 We should probably use map().
863 lines: List of error lines, each a string
865 New list with only interesting lines included
869 if not self.re_make_err.search(line):
870 out_lines.append(line)
873 def ReadFuncSizes(self, fname, fd):
874 """Read function sizes from the output of 'nm'
877 fd: File containing data to read
878 fname: Filename we are reading from (just for errors)
881 Dictionary containing size of each function in bytes, indexed by
885 for line in fd.readlines():
887 size, type, name = line[:-1].split()
889 print "Invalid line in file '%s': '%s'" % (fname, line[:-1])
892 # function names begin with '.' on 64-bit powerpc
894 name = 'static.' + name.split('.')[0]
895 sym[name] = sym.get(name, 0) + int(size, 16)
898 def GetBuildOutcome(self, commit_upto, target, read_func_sizes):
899 """Work out the outcome of a build.
902 commit_upto: Commit number to check (0..n-1)
903 target: Target board to check
904 read_func_sizes: True to read function size information
909 done_file = self.GetDoneFile(commit_upto, target)
910 sizes_file = self.GetSizesFile(commit_upto, target)
913 if os.path.exists(done_file):
914 with open(done_file, 'r') as fd:
915 return_code = int(fd.readline())
917 err_file = self.GetErrFile(commit_upto, target)
918 if os.path.exists(err_file):
919 with open(err_file, 'r') as fd:
920 err_lines = self.FilterErrors(fd.readlines())
922 # Decide whether the build was ok, failed or created warnings
930 # Convert size information to our simple format
931 if os.path.exists(sizes_file):
932 with open(sizes_file, 'r') as fd:
933 for line in fd.readlines():
934 values = line.split()
937 rodata = int(values[6], 16)
939 'all' : int(values[0]) + int(values[1]) +
941 'text' : int(values[0]) - rodata,
942 'data' : int(values[1]),
943 'bss' : int(values[2]),
946 sizes[values[5]] = size_dict
949 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
950 for fname in glob.glob(pattern):
951 with open(fname, 'r') as fd:
952 dict_name = os.path.basename(fname).replace('.sizes',
954 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
956 return Builder.Outcome(rc, err_lines, sizes, func_sizes)
958 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {})
960 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes):
961 """Calculate a summary of the results of building a commit.
964 board_selected: Dict containing boards to summarise
965 commit_upto: Commit number to summarize (0..self.count-1)
966 read_func_sizes: True to read function size information
970 Dict containing boards which passed building this commit.
971 keyed by board.target
972 List containing a summary of error/warning lines
975 err_lines_summary = []
977 for board in boards_selected.itervalues():
978 outcome = self.GetBuildOutcome(commit_upto, board.target,
980 board_dict[board.target] = outcome
981 for err in outcome.err_lines:
982 if err and not err.rstrip() in err_lines_summary:
983 err_lines_summary.append(err.rstrip())
984 return board_dict, err_lines_summary
986 def AddOutcome(self, board_dict, arch_list, changes, char, color):
987 """Add an output to our list of outcomes for each architecture
989 This simple function adds failing boards (changes) to the
990 relevant architecture string, so we can print the results out
991 sorted by architecture.
994 board_dict: Dict containing all boards
995 arch_list: Dict keyed by arch name. Value is a string containing
996 a list of board names which failed for that arch.
997 changes: List of boards to add to arch_list
998 color: terminal.Colour object
1001 for target in changes:
1002 if target in board_dict:
1003 arch = board_dict[target].arch
1006 str = self.col.Color(color, ' ' + target)
1007 if not arch in done_arch:
1008 str = self.col.Color(color, char) + ' ' + str
1009 done_arch[arch] = True
1010 if not arch in arch_list:
1011 arch_list[arch] = str
1013 arch_list[arch] += str
1016 def ColourNum(self, num):
1017 color = self.col.RED if num > 0 else self.col.GREEN
1020 return self.col.Color(color, str(num))
1022 def ResetResultSummary(self, board_selected):
1023 """Reset the results summary ready for use.
1025 Set up the base board list to be all those selected, and set the
1026 error lines to empty.
1028 Following this, calls to PrintResultSummary() will use this
1029 information to work out what has changed.
1032 board_selected: Dict containing boards to summarise, keyed by
1035 self._base_board_dict = {}
1036 for board in board_selected:
1037 self._base_board_dict[board] = Builder.Outcome(0, [], [], {})
1038 self._base_err_lines = []
1040 def PrintFuncSizeDetail(self, fname, old, new):
1041 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
1042 delta, common = [], {}
1049 if name not in common:
1052 delta.append([-old[name], name])
1055 if name not in common:
1058 delta.append([new[name], name])
1061 diff = new.get(name, 0) - old.get(name, 0)
1063 grow, up = grow + 1, up + diff
1065 shrink, down = shrink + 1, down - diff
1066 delta.append([diff, name])
1071 args = [add, -remove, grow, -shrink, up, -down, up - down]
1074 args = [self.ColourNum(x) for x in args]
1076 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
1077 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
1078 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
1080 for diff, name in delta:
1082 color = self.col.RED if diff > 0 else self.col.GREEN
1083 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
1084 old.get(name, '-'), new.get(name,'-'), diff)
1085 print self.col.Color(color, msg)
1088 def PrintSizeDetail(self, target_list, show_bloat):
1089 """Show details size information for each board
1092 target_list: List of targets, each a dict containing:
1093 'target': Target name
1094 'total_diff': Total difference in bytes across all areas
1095 <part_name>: Difference for that part
1096 show_bloat: Show detail for each function
1098 targets_by_diff = sorted(target_list, reverse=True,
1099 key=lambda x: x['_total_diff'])
1100 for result in targets_by_diff:
1101 printed_target = False
1102 for name in sorted(result):
1104 if name.startswith('_'):
1107 color = self.col.RED if diff > 0 else self.col.GREEN
1108 msg = ' %s %+d' % (name, diff)
1109 if not printed_target:
1110 print '%10s %-15s:' % ('', result['_target']),
1111 printed_target = True
1112 print self.col.Color(color, msg),
1116 target = result['_target']
1117 outcome = result['_outcome']
1118 base_outcome = self._base_board_dict[target]
1119 for fname in outcome.func_sizes:
1120 self.PrintFuncSizeDetail(fname,
1121 base_outcome.func_sizes[fname],
1122 outcome.func_sizes[fname])
1125 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
1127 """Print a summary of image sizes broken down by section.
1129 The summary takes the form of one line per architecture. The
1130 line contains deltas for each of the sections (+ means the section
1131 got bigger, - means smaller). The nunmbers are the average number
1132 of bytes that a board in this section increased by.
1135 powerpc: (622 boards) text -0.0
1136 arm: (285 boards) text -0.0
1137 nds32: (3 boards) text -8.0
1140 board_selected: Dict containing boards to summarise, keyed by
1142 board_dict: Dict containing boards for which we built this
1143 commit, keyed by board.target. The value is an Outcome object.
1144 show_detail: Show detail for each board
1145 show_bloat: Show detail for each function
1150 # Calculate changes in size for different image parts
1151 # The previous sizes are in Board.sizes, for each board
1152 for target in board_dict:
1153 if target not in board_selected:
1155 base_sizes = self._base_board_dict[target].sizes
1156 outcome = board_dict[target]
1157 sizes = outcome.sizes
1159 # Loop through the list of images, creating a dict of size
1160 # changes for each image/part. We end up with something like
1161 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1162 # which means that U-Boot data increased by 5 bytes and SPL
1163 # text decreased by 4.
1164 err = {'_target' : target}
1166 if image in base_sizes:
1167 base_image = base_sizes[image]
1168 # Loop through the text, data, bss parts
1169 for part in sorted(sizes[image]):
1170 diff = sizes[image][part] - base_image[part]
1173 if image == 'u-boot':
1176 name = image + ':' + part
1178 arch = board_selected[target].arch
1179 if not arch in arch_count:
1180 arch_count[arch] = 1
1182 arch_count[arch] += 1
1184 pass # Only add to our list when we have some stats
1185 elif not arch in arch_list:
1186 arch_list[arch] = [err]
1188 arch_list[arch].append(err)
1190 # We now have a list of image size changes sorted by arch
1191 # Print out a summary of these
1192 for arch, target_list in arch_list.iteritems():
1193 # Get total difference for each type
1195 for result in target_list:
1197 for name, diff in result.iteritems():
1198 if name.startswith('_'):
1202 totals[name] += diff
1205 result['_total_diff'] = total
1206 result['_outcome'] = board_dict[result['_target']]
1208 count = len(target_list)
1209 printed_arch = False
1210 for name in sorted(totals):
1213 # Display the average difference in this name for this
1215 avg_diff = float(diff) / count
1216 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1217 msg = ' %s %+1.1f' % (name, avg_diff)
1218 if not printed_arch:
1219 print '%10s: (for %d/%d boards)' % (arch, count,
1222 print self.col.Color(color, msg),
1227 self.PrintSizeDetail(target_list, show_bloat)
1230 def PrintResultSummary(self, board_selected, board_dict, err_lines,
1231 show_sizes, show_detail, show_bloat):
1232 """Compare results with the base results and display delta.
1234 Only boards mentioned in board_selected will be considered. This
1235 function is intended to be called repeatedly with the results of
1236 each commit. It therefore shows a 'diff' between what it saw in
1237 the last call and what it sees now.
1240 board_selected: Dict containing boards to summarise, keyed by
1242 board_dict: Dict containing boards for which we built this
1243 commit, keyed by board.target. The value is an Outcome object.
1244 err_lines: A list of errors for this commit, or [] if there is
1245 none, or we don't want to print errors
1246 show_sizes: Show image size deltas
1247 show_detail: Show detail for each board
1248 show_bloat: Show detail for each function
1250 better = [] # List of boards fixed since last commit
1251 worse = [] # List of new broken boards since last commit
1252 new = [] # List of boards that didn't exist last time
1253 unknown = [] # List of boards that were not built
1255 for target in board_dict:
1256 if target not in board_selected:
1259 # If the board was built last time, add its outcome to a list
1260 if target in self._base_board_dict:
1261 base_outcome = self._base_board_dict[target].rc
1262 outcome = board_dict[target]
1263 if outcome.rc == OUTCOME_UNKNOWN:
1264 unknown.append(target)
1265 elif outcome.rc < base_outcome:
1266 better.append(target)
1267 elif outcome.rc > base_outcome:
1268 worse.append(target)
1272 # Get a list of errors that have appeared, and disappeared
1275 for line in err_lines:
1276 if line not in self._base_err_lines:
1277 worse_err.append('+' + line)
1278 for line in self._base_err_lines:
1279 if line not in err_lines:
1280 better_err.append('-' + line)
1282 # Display results by arch
1283 if better or worse or unknown or new or worse_err or better_err:
1285 self.AddOutcome(board_selected, arch_list, better, '',
1287 self.AddOutcome(board_selected, arch_list, worse, '+',
1289 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
1290 if self._show_unknown:
1291 self.AddOutcome(board_selected, arch_list, unknown, '?',
1293 for arch, target_list in arch_list.iteritems():
1294 print '%10s: %s' % (arch, target_list)
1296 print self.col.Color(self.col.GREEN, '\n'.join(better_err))
1298 print self.col.Color(self.col.RED, '\n'.join(worse_err))
1301 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1304 # Save our updated information for the next call to this function
1305 self._base_board_dict = board_dict
1306 self._base_err_lines = err_lines
1308 # Get a list of boards that did not get built, if needed
1310 for board in board_selected:
1311 if not board in board_dict:
1312 not_built.append(board)
1314 print "Boards not built (%d): %s" % (len(not_built),
1315 ', '.join(not_built))
1318 def ShowSummary(self, commits, board_selected, show_errors, show_sizes,
1319 show_detail, show_bloat):
1320 """Show a build summary for U-Boot for a given board list.
1322 Reset the result summary, then repeatedly call GetResultSummary on
1323 each commit's results, then display the differences we see.
1326 commit: Commit objects to summarise
1327 board_selected: Dict containing boards to summarise
1328 show_errors: Show errors that occured
1329 show_sizes: Show size deltas
1330 show_detail: Show detail for each board
1331 show_bloat: Show detail for each function
1333 self.commit_count = len(commits) if commits else 1
1334 self.commits = commits
1335 self.ResetResultSummary(board_selected)
1337 for commit_upto in range(0, self.commit_count, self._step):
1338 board_dict, err_lines = self.GetResultSummary(board_selected,
1339 commit_upto, read_func_sizes=show_bloat)
1341 msg = '%02d: %s' % (commit_upto + 1,
1342 commits[commit_upto].subject)
1345 print self.col.Color(self.col.BLUE, msg)
1346 self.PrintResultSummary(board_selected, board_dict,
1347 err_lines if show_errors else [], show_sizes, show_detail,
1351 def SetupBuild(self, board_selected, commits):
1352 """Set up ready to start a build.
1355 board_selected: Selected boards to build
1356 commits: Selected commits to build
1358 # First work out how many commits we will build
1359 count = (self.commit_count + self._step - 1) / self._step
1360 self.count = len(board_selected) * count
1361 self.upto = self.warned = self.fail = 0
1362 self._timestamps = collections.deque()
1364 def BuildBoardsForCommit(self, board_selected, keep_outputs):
1365 """Build all boards for a single commit"""
1366 self.SetupBuild(board_selected)
1367 self.count = len(board_selected)
1368 for brd in board_selected.itervalues():
1372 job.keep_outputs = keep_outputs
1376 self.out_queue.join()
1380 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs):
1381 """Build all boards for all commits (non-incremental)"""
1382 self.commit_count = len(commits)
1384 self.ResetResultSummary(board_selected)
1385 for self.commit_upto in range(self.commit_count):
1386 self.SelectCommit(commits[self.commit_upto])
1387 self.SelectOutputDir()
1388 Mkdir(self.output_dir)
1390 self.BuildBoardsForCommit(board_selected, keep_outputs)
1391 board_dict, err_lines = self.GetResultSummary()
1392 self.PrintResultSummary(board_selected, board_dict,
1393 err_lines if show_errors else [])
1395 if self.already_done:
1396 print '%d builds already done' % self.already_done
1398 def GetThreadDir(self, thread_num):
1399 """Get the directory path to the working dir for a thread.
1402 thread_num: Number of thread to check.
1404 return os.path.join(self._working_dir, '%02d' % thread_num)
1406 def _PrepareThread(self, thread_num, setup_git):
1407 """Prepare the working directory for a thread.
1409 This clones or fetches the repo into the thread's work directory.
1412 thread_num: Thread number (0, 1, ...)
1413 setup_git: True to set up a git repo clone
1415 thread_dir = self.GetThreadDir(thread_num)
1417 git_dir = os.path.join(thread_dir, '.git')
1419 # Clone the repo if it doesn't already exist
1420 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1421 # we have a private index but uses the origin repo's contents?
1422 if setup_git and self.git_dir:
1423 src_dir = os.path.abspath(self.git_dir)
1424 if os.path.exists(git_dir):
1425 gitutil.Fetch(git_dir, thread_dir)
1427 print 'Cloning repo for thread %d' % thread_num
1428 gitutil.Clone(src_dir, thread_dir)
1430 def _PrepareWorkingSpace(self, max_threads, setup_git):
1431 """Prepare the working directory for use.
1433 Set up the git repo for each thread.
1436 max_threads: Maximum number of threads we expect to need.
1437 setup_git: True to set up a git repo clone
1439 Mkdir(self._working_dir)
1440 for thread in range(max_threads):
1441 self._PrepareThread(thread, setup_git)
1443 def _PrepareOutputSpace(self):
1444 """Get the output directories ready to receive files.
1446 We delete any output directories which look like ones we need to
1447 create. Having left over directories is confusing when the user wants
1448 to check the output manually.
1451 for commit_upto in range(self.commit_count):
1452 dir_list.append(self._GetOutputDir(commit_upto))
1454 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1455 if dirname not in dir_list:
1456 shutil.rmtree(dirname)
1458 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs):
1459 """Build all commits for a list of boards
1462 commits: List of commits to be build, each a Commit object
1463 boards_selected: Dict of selected boards, key is target name,
1464 value is Board object
1465 show_errors: True to show summarised error/warning info
1466 keep_outputs: True to save build output files
1468 self.commit_count = len(commits) if commits else 1
1469 self.commits = commits
1471 self.ResetResultSummary(board_selected)
1472 Mkdir(self.base_dir)
1473 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1474 commits is not None)
1475 self._PrepareOutputSpace()
1476 self.SetupBuild(board_selected, commits)
1477 self.ProcessResult(None)
1479 # Create jobs to build all commits for each board
1480 for brd in board_selected.itervalues():
1483 job.commits = commits
1484 job.keep_outputs = keep_outputs
1485 job.step = self._step
1488 # Wait until all jobs are started
1491 # Wait until we have processed all output
1492 self.out_queue.join()