Skip to content

Include cache fixes and refactorings #236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
66cdcc0
Fix past-end-of-cache handling in includeCache.ExpectFile
matthijskooijman Jun 6, 2017
f69a8cc
Convert IncludesFinderWithRegExp to a normal function
matthijskooijman Jun 6, 2017
be08809
Convert GCCPreprocRunner(ForDiscoveringIncludes) to a normal function
matthijskooijman Jun 6, 2017
85d3624
Refactor path generation for ctags_target_for_gcc_minus_e.cpp
matthijskooijman Jun 6, 2017
a14e0ef
Pass FileToRead to ReadFileAndStoreInContext explicitly
matthijskooijman Jun 6, 2017
73bae11
Remove GCCPreprocSourceSaver
matthijskooijman Jun 6, 2017
b861a84
execSizeRecipe: Fix typo in method name
matthijskooijman Jun 16, 2017
108e65f
Pass types.Context down into compilation helpers
matthijskooijman Jun 16, 2017
0f1baa5
Show the sizer commandline in verbose mode
matthijskooijman Jun 16, 2017
2254d74
Show stdout of preproc commands in verbose mode
matthijskooijman Jun 16, 2017
611f07a
Do not ignore command errors in ExecRecipeCollectStdErr
matthijskooijman Jun 16, 2017
5eb37bc
Let ExecRecipeCollectStdErr return []byte for stderr
matthijskooijman Jun 16, 2017
ca9008e
Improve error handling in include detection
matthijskooijman Jun 16, 2017
60265b5
Merge ExecRecipeCollectStdErr into ExecRecipe
matthijskooijman Jun 16, 2017
31e42b7
Merge some duplicate code into prepareGCCPreprocRecipeProperties
matthijskooijman Jun 16, 2017
b03ecab
Let utils.ExecCommand print the command in verbose mode
matthijskooijman Jun 16, 2017
9abc875
Fix removal of -MMD option when running the preprocessor
matthijskooijman Jun 16, 2017
788cecd
Use utils.ExecCommand for running ctags
matthijskooijman Jun 16, 2017
cc978ec
Pass Context to ObjFileIsUpToDate
matthijskooijman Nov 30, 2017
e92bf65
Let ObjFileIsUpToDate output verbose debug output
matthijskooijman Mar 2, 2017
d2d49e6
ContainerFindIncludes: Add some temporary variables
matthijskooijman Mar 2, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 64 additions & 60 deletions builder_utils/utils.go
Original file line number Diff line number Diff line change
@@ -30,8 +30,6 @@
package builder_utils

import (
"bytes"
"fmt"
"io"
"os"
"os/exec"
@@ -40,12 +38,13 @@ import (

"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/i18n"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"github.com/arduino/go-properties-map"
)

func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
objectFiles, err := CompileFiles(objectFiles, sourcePath, false, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
func CompileFilesRecursive(ctx *types.Context, objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) {
objectFiles, err := CompileFiles(ctx, objectFiles, sourcePath, false, buildPath, buildProperties, includes)
if err != nil {
return nil, i18n.WrapError(err)
}
@@ -56,7 +55,7 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st
}

for _, folder := range folders {
objectFiles, err = CompileFilesRecursive(objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes, verbose, warningsLevel, logger)
objectFiles, err = CompileFilesRecursive(ctx, objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes)
if err != nil {
return nil, i18n.WrapError(err)
}
@@ -65,28 +64,28 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st
return objectFiles, nil
}

func CompileFiles(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
objectFiles, err := compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN, verbose, warningsLevel, logger)
func CompileFiles(ctx *types.Context, objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) {
objectFiles, err := compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN)
if err != nil {
return nil, i18n.WrapError(err)
}
objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN, verbose, warningsLevel, logger)
objectFiles, err = compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN)
if err != nil {
return nil, i18n.WrapError(err)
}
objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN, verbose, warningsLevel, logger)
objectFiles, err = compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN)
if err != nil {
return nil, i18n.WrapError(err)
}
return objectFiles, nil
}

func compileFilesWithExtensionWithRecipe(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
func compileFilesWithExtensionWithRecipe(ctx *types.Context, objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string) ([]string, error) {
sources, err := findFilesInFolder(sourcePath, extension, recurse)
if err != nil {
return nil, i18n.WrapError(err)
}
return compileFilesWithRecipe(objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
return compileFilesWithRecipe(ctx, objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe)
}

func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]string, error) {
@@ -145,9 +144,9 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) {
return sources, nil
}

func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
func compileFilesWithRecipe(ctx *types.Context, objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string) ([]string, error) {
for _, source := range sources {
objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe)
if err != nil {
return nil, i18n.WrapError(err)
}
@@ -157,9 +156,10 @@ func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []s
return objectFiles, nil
}

func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) {
func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string) (string, error) {
logger := ctx.GetLogger()
properties := buildProperties.Clone()
properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel]
properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel]
properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = source
relativeSource, err := filepath.Rel(sourcePath, source)
@@ -173,27 +173,33 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b
return "", i18n.WrapError(err)
}

objIsUpToDate, err := ObjFileIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d"))
objIsUpToDate, err := ObjFileIsUpToDate(ctx, properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d"))
if err != nil {
return "", i18n.WrapError(err)
}

if !objIsUpToDate {
_, err = ExecRecipe(properties, recipe, false, verbose, verbose, logger)
_, _, err = ExecRecipe(ctx, properties, recipe, false, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show)
if err != nil {
return "", i18n.WrapError(err)
}
} else if verbose {
} else if ctx.Verbose {
logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, properties[constants.BUILD_PROPERTIES_OBJECT_FILE])
}

return properties[constants.BUILD_PROPERTIES_OBJECT_FILE], nil
}

func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, error) {
func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFile string) (bool, error) {
sourceFile = filepath.Clean(sourceFile)
objectFile = filepath.Clean(objectFile)
dependencyFile = filepath.Clean(dependencyFile)
logger := ctx.GetLogger()
debugLevel := ctx.DebugLevel

if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Checking previous results for {0} (result = {1}, dep = {2})", sourceFile, objectFile, dependencyFile)
}

sourceFileStat, err := os.Stat(sourceFile)
if err != nil {
@@ -203,6 +209,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err
objectFileStat, err := os.Stat(objectFile)
if err != nil {
if os.IsNotExist(err) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", objectFile)
}
return false, nil
} else {
return false, i18n.WrapError(err)
@@ -212,16 +221,25 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err
dependencyFileStat, err := os.Stat(dependencyFile)
if err != nil {
if os.IsNotExist(err) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", dependencyFile)
}
return false, nil
} else {
return false, i18n.WrapError(err)
}
}

if sourceFileStat.ModTime().After(objectFileStat.ModTime()) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, objectFile)
}
return false, nil
}
if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, dependencyFile)
}
return false, nil
}

@@ -241,10 +259,16 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err

firstRow := rows[0]
if !strings.HasSuffix(firstRow, ":") {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "No colon in first line of depfile")
}
return false, nil
}
objFileInDepFile := firstRow[:len(firstRow)-1]
if objFileInDepFile != objectFile {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Depfile is about different file: {0}", objFileInDepFile)
}
return false, nil
}

@@ -254,12 +278,22 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err
if err != nil && !os.IsNotExist(err) {
// There is probably a parsing error of the dep file
// Ignore the error and trigger a full rebuild anyway
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Failed to read: {0}", row)
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, i18n.WrapError(err).Error())
}
return false, nil
}
if os.IsNotExist(err) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", row)
}
return false, nil
}
if depStat.ModTime().After(objectFileStat.ModTime()) {
if debugLevel >= 20 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", row, objectFile)
}
return false, nil
}
}
@@ -309,7 +343,8 @@ func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string)
return true
}

func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) {
func ArchiveCompiledFiles(ctx *types.Context, buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map) (string, error) {
logger := ctx.GetLogger()
archiveFilePath := filepath.Join(buildPath, archiveFile)

rebuildArchive := false
@@ -332,7 +367,7 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st
return "", i18n.WrapError(err)
}
} else {
if verbose {
if ctx.Verbose {
logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, archiveFilePath)
}
return archiveFilePath, nil
@@ -345,7 +380,7 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st
properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = archiveFilePath
properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = objectFile

_, err := ExecRecipe(properties, constants.RECIPE_AR_PATTERN, false, verbose, verbose, logger)
_, _, err := ExecRecipe(ctx, properties, constants.RECIPE_AR_PATTERN, false, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show)
if err != nil {
return "", i18n.WrapError(err)
}
@@ -354,28 +389,17 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st
return archiveFilePath, nil
}

func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) ([]byte, error) {
command, err := PrepareCommandForRecipe(properties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger)
// See util.ExecCommand for stdout/stderr arguments
func ExecRecipe(ctx *types.Context, buildProperties properties.Map, recipe string, removeUnsetProperties bool, stdout int, stderr int) ([]byte, []byte, error) {
command, err := PrepareCommandForRecipe(ctx, buildProperties, recipe, removeUnsetProperties)
if err != nil {
return nil, i18n.WrapError(err)
return nil, nil, i18n.WrapError(err)
}

if echoOutput {
command.Stdout = os.Stdout
}

command.Stderr = os.Stderr

if echoOutput {
err := command.Run()
return nil, i18n.WrapError(err)
}

bytes, err := command.Output()
return bytes, i18n.WrapError(err)
return utils.ExecCommand(ctx, command, stdout, stderr)
}

func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (*exec.Cmd, error) {
func PrepareCommandForRecipe(ctx *types.Context, buildProperties properties.Map, recipe string, removeUnsetProperties bool) (*exec.Cmd, error) {
logger := ctx.GetLogger()
pattern := buildProperties[recipe]
if pattern == constants.EMPTY_STRING {
return nil, i18n.ErrorfWithLogger(logger, constants.MSG_PATTERN_MISSING, recipe)
@@ -392,29 +416,9 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo
return nil, i18n.WrapError(err)
}

if echoCommandLine {
fmt.Println(commandLine)
}

return command, nil
}

func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (string, error) {
command, err := PrepareCommandForRecipe(buildProperties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger)
if err != nil {
return "", i18n.WrapError(err)
}

buffer := &bytes.Buffer{}
command.Stderr = buffer
command.Run()
return string(buffer.Bytes()), nil
}

func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) {
buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1)
}

// CopyFile copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
17 changes: 15 additions & 2 deletions container_add_prototypes.go
Original file line number Diff line number Diff line change
@@ -35,15 +35,28 @@ import (
"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/i18n"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
)

type ContainerAddPrototypes struct{}

func (s *ContainerAddPrototypes) Run(ctx *types.Context) error {
sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp")

// Generate the full pathname for the preproc output file
err := utils.EnsureFolderExists(ctx.PreprocPath)
if err != nil {
return i18n.WrapError(err)
}
targetFilePath := filepath.Join(ctx.PreprocPath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E)

// Run preprocessor
err = GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders)
if err != nil {
return i18n.WrapError(err)
}
commands := []types.Command{
&GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders},
&ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE},
&ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SourceGccMinusE},
&FilterSketchSource{Source: &ctx.SourceGccMinusE},
&CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
&CTagsRunner{},
52 changes: 39 additions & 13 deletions container_find_includes.go
Original file line number Diff line number Diff line change
@@ -110,6 +110,7 @@ import (
"encoding/json"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"time"

@@ -118,6 +119,8 @@ import (
"github.com/arduino/arduino-builder/i18n"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"

"github.com/go-errors/errors"
)

type ContainerFindIncludes struct{}
@@ -213,7 +216,7 @@ func (cache *includeCache) Next() includeCacheEntry {
// not, or no entry is available, the cache is invalidated. Does not
// advance the cache.
func (cache *includeCache) ExpectFile(sourcefile string) {
if cache.valid && cache.next < len(cache.entries) && cache.Next().Sourcefile != sourcefile {
if cache.valid && (cache.next >= len(cache.entries) || cache.Next().Sourcefile != sourcefile) {
cache.valid = false
cache.entries = cache.entries[:cache.next]
}
@@ -289,6 +292,8 @@ func writeCache(cache *includeCache, path string) error {

func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error {
sourcePath := sourceFile.SourcePath(ctx)
depPath := sourceFile.DepfilePath(ctx)
objPath := sourceFile.ObjectPath(ctx)
targetFilePath := utils.NULLFile()

// TODO: This should perhaps also compare against the
@@ -303,7 +308,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
// TODO: This reads the dependency file, but the actual building
// does it again. Should the result be somehow cached? Perhaps
// remove the object file if it is found to be stale?
unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx))
unchanged, err := builder_utils.ObjFileIsUpToDate(ctx, sourcePath, objPath, depPath)
if err != nil {
return i18n.WrapError(err)
}
@@ -317,23 +322,33 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
if library, ok := sourceFile.Origin.(*types.Library); ok && library.UtilityFolder != "" {
includes = append(includes, library.UtilityFolder)
}
var preproc_err error
var preproc_stderr []byte
if unchanged && cache.valid {
include = cache.Next().Include
if first && ctx.Verbose {
ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_CACHED_INCLUDES, sourcePath)
}
} else {
commands := []types.Command{
&GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, TargetFilePath: targetFilePath, Includes: includes},
&IncludesFinderWithRegExp{Source: &ctx.SourceGccMinusE},
}
for _, command := range commands {
err := runCommand(ctx, command)
if err != nil {
return i18n.WrapError(err)
preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes)
// Unwrap error and see if it is an ExitError.
_, is_exit_error := i18n.UnwrapError(preproc_err).(*exec.ExitError)
if preproc_err == nil {
// Preprocessor successful, done
include = ""
} else if !is_exit_error || preproc_stderr == nil {
// Ignore ExitErrors (e.g. gcc returning
// non-zero status), but bail out on
// other errors
return i18n.WrapError(preproc_err)
} else {
include = IncludesFinderWithRegExp(ctx, string(preproc_stderr))
if include == "" {
// No include found? Bail out.
os.Stderr.Write(preproc_stderr)
return i18n.WrapError(preproc_err)
}
}
include = ctx.IncludeJustFound
}

if include == "" {
@@ -345,8 +360,19 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
library := ResolveLibrary(ctx, include)
if library == nil {
// Library could not be resolved, show error
err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: includes})
return i18n.WrapError(err)
if preproc_err == nil || preproc_stderr == nil {
// Filename came from cache, so run preprocessor to obtain error to show
preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes)
if preproc_err == nil {
// If there is a missing #include in the cache, but running
// gcc does not reproduce that, there is something wrong.
// Returning an error here will cause the cache to be
// deleted, so hopefully the next compilation will succeed.
return errors.New("Internal error in cache")
}
}
os.Stderr.Write(preproc_stderr)
return i18n.WrapError(preproc_err)
}

// Add this library to the list of libraries, the
9 changes: 1 addition & 8 deletions ctags_runner.go
Original file line number Diff line number Diff line change
@@ -30,8 +30,6 @@
package builder

import (
"fmt"

"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/ctags"
"github.com/arduino/arduino-builder/i18n"
@@ -61,12 +59,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error {
return i18n.WrapError(err)
}

verbose := ctx.Verbose
if verbose {
fmt.Println(commandLine)
}

sourceBytes, err := command.Output()
sourceBytes, _, err := utils.ExecCommand(ctx, command /* stdout */, utils.Capture /* stderr */, utils.Ignore)
if err != nil {
return i18n.WrapError(err)
}
80 changes: 26 additions & 54 deletions gcc_preproc_runner.go
Original file line number Diff line number Diff line change
@@ -30,95 +30,67 @@
package builder

import (
"path/filepath"
"os/exec"
"strings"

"github.com/arduino/arduino-builder/builder_utils"
"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/i18n"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"github.com/arduino/go-properties-map"
)

type GCCPreprocRunner struct {
SourceFilePath string
TargetFileName string
Includes []string
}

func (s *GCCPreprocRunner) Run(ctx *types.Context) error {
properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFileName, s.Includes)
func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) error {
cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes)
if err != nil {
return i18n.WrapError(err)
}

if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING {
//generate PREPROC_MACROS from RECIPE_CPP_PATTERN
properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN])
}

verbose := ctx.Verbose
logger := ctx.GetLogger()
_, err = builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger)
_, _, err = utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show)
if err != nil {
return i18n.WrapError(err)
}

ctx.FileToRead = targetFilePath

return nil
}

type GCCPreprocRunnerForDiscoveringIncludes struct {
SourceFilePath string
TargetFilePath string
Includes []string
}

func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error {
properties, _, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFilePath, s.Includes)
func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) ([]byte, error) {
cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes)
if err != nil {
return i18n.WrapError(err)
return nil, i18n.WrapError(err)
}

verbose := ctx.Verbose
logger := ctx.GetLogger()

if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING {
//generate PREPROC_MACROS from RECIPE_CPP_PATTERN
properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN])
}

stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger)
_, stderr, err := utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Capture)
if err != nil {
return i18n.WrapError(err)
return stderr, i18n.WrapError(err)
}

ctx.SourceGccMinusE = string(stderr)

return nil
return stderr, nil
}

func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) {
if targetFilePath != utils.NULLFile() {
preprocPath := ctx.PreprocPath
err := utils.EnsureFolderExists(preprocPath)
if err != nil {
return nil, "", i18n.WrapError(err)
}
targetFilePath = filepath.Join(preprocPath, targetFilePath)
}

func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (*exec.Cmd, error) {
properties := ctx.BuildProperties.Clone()
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFilePath
properties[constants.BUILD_PROPERTIES_PREPROCESSED_FILE_PATH] = targetFilePath

includes = utils.Map(includes, utils.WrapWithHyphenI)
properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)
builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties)

return properties, targetFilePath, nil
if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING {
//generate PREPROC_MACROS from RECIPE_CPP_PATTERN
properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN])
}

cmd, err := builder_utils.PrepareCommandForRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true)
if err != nil {
return nil, i18n.WrapError(err)
}

// Remove -MMD argument if present. Leaving it will make gcc try
// to create a /dev/null.d dependency file, which won't work.
cmd.Args = utils.Filter(cmd.Args, func(a string) bool { return a != "-MMD" })

return cmd, nil
}

func GeneratePreprocPatternFromCompile(compilePattern string) string {
53 changes: 0 additions & 53 deletions gcc_preproc_source_saver.go

This file was deleted.

11 changes: 11 additions & 0 deletions i18n/errors.go
Original file line number Diff line number Diff line change
@@ -18,3 +18,14 @@ func WrapError(err error) error {
}
return errors.Wrap(err, 0)
}

func UnwrapError(err error) error {
// Perhaps go-errors can do this already in later versions?
// See https://github.com/go-errors/errors/issues/14
switch e := err.(type) {
case *errors.Error:
return e.Err
default:
return err
}
}
14 changes: 3 additions & 11 deletions includes_finder_with_regexp.go
Original file line number Diff line number Diff line change
@@ -37,21 +37,13 @@ import (

var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]")

type IncludesFinderWithRegExp struct {
Source *string
}

func (s *IncludesFinderWithRegExp) Run(ctx *types.Context) error {
source := *s.Source

func IncludesFinderWithRegExp(ctx *types.Context, source string) string {
match := INCLUDE_REGEXP.FindStringSubmatch(source)
if match != nil {
ctx.IncludeJustFound = strings.TrimSpace(match[1])
return strings.TrimSpace(match[1])
} else {
ctx.IncludeJustFound = findIncludeForOldCompilers(source)
return findIncludeForOldCompilers(source)
}

return nil
}

func findIncludeForOldCompilers(source string) string {
18 changes: 8 additions & 10 deletions phases/core_builder.go
Original file line number Diff line number Diff line change
@@ -46,9 +46,6 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
coreBuildPath := ctx.CoreBuildPath
coreBuildCachePath := ctx.CoreBuildCachePath
buildProperties := ctx.BuildProperties
verbose := ctx.Verbose
warningsLevel := ctx.WarningsLevel
logger := ctx.GetLogger()

err := utils.EnsureFolderExists(coreBuildPath)
if err != nil {
@@ -62,7 +59,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
}
}

archiveFile, objectFiles, err := compileCore(coreBuildPath, coreBuildCachePath, buildProperties, verbose, warningsLevel, logger)
archiveFile, objectFiles, err := compileCore(ctx, coreBuildPath, coreBuildCachePath, buildProperties)
if err != nil {
return i18n.WrapError(err)
}
@@ -73,7 +70,8 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
return nil
}

func compileCore(buildPath string, buildCachePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) {
func compileCore(ctx *types.Context, buildPath string, buildCachePath string, buildProperties properties.Map) (string, []string, error) {
logger := ctx.GetLogger()
coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH]
variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH]

@@ -90,7 +88,7 @@ func compileCore(buildPath string, buildCachePath string, buildProperties proper

variantObjectFiles := []string{}
if variantFolder != constants.EMPTY_STRING {
variantObjectFiles, err = builder_utils.CompileFiles(variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
variantObjectFiles, err = builder_utils.CompileFiles(ctx, variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes)
if err != nil {
return "", nil, i18n.WrapError(err)
}
@@ -107,26 +105,26 @@ func compileCore(buildPath string, buildCachePath string, buildProperties proper

if canUseArchivedCore {
// use archived core
if verbose {
if ctx.Verbose {
logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core: {0}", targetArchivedCore)
}
return targetArchivedCore, variantObjectFiles, nil
}
}

coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
coreObjectFiles, err := builder_utils.CompileFiles(ctx, []string{}, coreFolder, true, buildPath, buildProperties, includes)
if err != nil {
return "", nil, i18n.WrapError(err)
}

archiveFile, err := builder_utils.ArchiveCompiledFiles(buildPath, "core.a", coreObjectFiles, buildProperties, verbose, logger)
archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, buildPath, "core.a", coreObjectFiles, buildProperties)
if err != nil {
return "", nil, i18n.WrapError(err)
}

// archive core.a
if targetArchivedCore != "" {
if verbose {
if ctx.Verbose {
logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_ARCHIVING_CORE_CACHE, targetArchivedCore)
}
builder_utils.CopyFile(archiveFile, targetArchivedCore)
22 changes: 10 additions & 12 deletions phases/libraries_builder.go
Original file line number Diff line number Diff line change
@@ -52,16 +52,13 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error {
includes := ctx.IncludeFolders
includes = utils.Map(includes, utils.WrapWithHyphenI)
libraries := ctx.ImportedLibraries
verbose := ctx.Verbose
warningsLevel := ctx.WarningsLevel
logger := ctx.GetLogger()

err := utils.EnsureFolderExists(librariesBuildPath)
if err != nil {
return i18n.WrapError(err)
}

objectFiles, err := compileLibraries(libraries, librariesBuildPath, buildProperties, includes, verbose, warningsLevel, logger)
objectFiles, err := compileLibraries(ctx, libraries, librariesBuildPath, buildProperties, includes)
if err != nil {
return i18n.WrapError(err)
}
@@ -99,10 +96,10 @@ func fixLDFLAGforPrecompiledLibraries(ctx *types.Context, libraries []*types.Lib
return nil
}

func compileLibraries(libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
func compileLibraries(ctx *types.Context, libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) {
objectFiles := []string{}
for _, library := range libraries {
libraryObjectFiles, err := compileLibrary(library, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
libraryObjectFiles, err := compileLibrary(ctx, library, buildPath, buildProperties, includes)
if err != nil {
return nil, i18n.WrapError(err)
}
@@ -113,8 +110,9 @@ func compileLibraries(libraries []*types.Library, buildPath string, buildPropert

}

func compileLibrary(library *types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
if verbose {
func compileLibrary(ctx *types.Context, library *types.Library, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) {
logger := ctx.GetLogger()
if ctx.Verbose {
logger.Println(constants.LOG_LEVEL_INFO, "Compiling library \"{0}\"", library.Name)
}
libraryBuildPath := filepath.Join(buildPath, library.Name)
@@ -144,12 +142,12 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr
}

if library.Layout == types.LIBRARY_RECURSIVE {
objectFiles, err = builder_utils.CompileFilesRecursive(objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger)
objectFiles, err = builder_utils.CompileFilesRecursive(ctx, objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes)
if err != nil {
return nil, i18n.WrapError(err)
}
if library.DotALinkage {
archiveFile, err := builder_utils.ArchiveCompiledFiles(libraryBuildPath, library.Name+".a", objectFiles, buildProperties, verbose, logger)
archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, libraryBuildPath, library.Name+".a", objectFiles, buildProperties)
if err != nil {
return nil, i18n.WrapError(err)
}
@@ -159,14 +157,14 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr
if library.UtilityFolder != "" {
includes = append(includes, utils.WrapWithHyphenI(library.UtilityFolder))
}
objectFiles, err = builder_utils.CompileFiles(objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger)
objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes)
if err != nil {
return nil, i18n.WrapError(err)
}

if library.UtilityFolder != "" {
utilityBuildPath := filepath.Join(libraryBuildPath, constants.LIBRARY_FOLDER_UTILITY)
objectFiles, err = builder_utils.CompileFiles(objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes, verbose, warningsLevel, logger)
objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes)
if err != nil {
return nil, i18n.WrapError(err)
}
11 changes: 4 additions & 7 deletions phases/linker.go
Original file line number Diff line number Diff line change
@@ -61,32 +61,29 @@ func (s *Linker) Run(ctx *types.Context) error {
}

buildProperties := ctx.BuildProperties
verbose := ctx.Verbose
warningsLevel := ctx.WarningsLevel
logger := ctx.GetLogger()

err = link(objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties, verbose, warningsLevel, logger)
err = link(ctx, objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties)
if err != nil {
return i18n.WrapError(err)
}

return nil
}

func link(objectFiles []string, coreDotARelPath string, coreArchiveFilePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) error {
func link(ctx *types.Context, objectFiles []string, coreDotARelPath string, coreArchiveFilePath string, buildProperties properties.Map) error {
optRelax := addRelaxTrickIfATMEGA2560(buildProperties)

objectFiles = utils.Map(objectFiles, wrapWithDoubleQuotes)
objectFileList := strings.Join(objectFiles, constants.SPACE)

properties := buildProperties.Clone()
properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] + optRelax
properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel]
properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel]
properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE] = coreDotARelPath
properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = coreArchiveFilePath
properties[constants.BUILD_PROPERTIES_OBJECT_FILES] = objectFileList

_, err := builder_utils.ExecRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, verbose, verbose, logger)
_, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_C_COMBINE_PATTERN, false, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show)
return err
}

17 changes: 8 additions & 9 deletions phases/sizer.go
Original file line number Diff line number Diff line change
@@ -38,6 +38,7 @@ import (
"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/i18n"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"github.com/arduino/go-properties-map"
)

@@ -52,22 +53,20 @@ func (s *Sizer) Run(ctx *types.Context) error {
}

buildProperties := ctx.BuildProperties
verbose := ctx.Verbose
warningsLevel := ctx.WarningsLevel
logger := ctx.GetLogger()

err := checkSize(buildProperties, verbose, warningsLevel, logger)
err := checkSize(ctx, buildProperties)
if err != nil {
return i18n.WrapError(err)
}

return nil
}

func checkSize(buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) error {
func checkSize(ctx *types.Context, buildProperties properties.Map) error {
logger := ctx.GetLogger()

properties := buildProperties.Clone()
properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel]
properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel]

maxTextSizeString := properties[constants.PROPERTY_UPLOAD_MAX_SIZE]
maxDataSizeString := properties[constants.PROPERTY_UPLOAD_MAX_DATA_SIZE]
@@ -89,7 +88,7 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin
}
}

textSize, dataSize, _, err := execSizeReceipe(properties, logger)
textSize, dataSize, _, err := execSizeRecipe(ctx, properties)
if err != nil {
logger.Println(constants.LOG_LEVEL_WARN, constants.MSG_SIZER_ERROR_NO_RULE)
return nil
@@ -127,8 +126,8 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin
return nil
}

func execSizeReceipe(properties properties.Map, logger i18n.Logger) (textSize int, dataSize int, eepromSize int, resErr error) {
out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, false, false, logger)
func execSizeRecipe(ctx *types.Context, properties properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) {
out, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_SIZE_PATTERN, false /* stdout */, utils.Capture /* stderr */, utils.Show)
if err != nil {
resErr = errors.New("Error while determining sketch size: " + err.Error())
return
7 changes: 2 additions & 5 deletions phases/sketch_builder.go
Original file line number Diff line number Diff line change
@@ -46,25 +46,22 @@ func (s *SketchBuilder) Run(ctx *types.Context) error {
buildProperties := ctx.BuildProperties
includes := ctx.IncludeFolders
includes = utils.Map(includes, utils.WrapWithHyphenI)
verbose := ctx.Verbose
warningsLevel := ctx.WarningsLevel
logger := ctx.GetLogger()

err := utils.EnsureFolderExists(sketchBuildPath)
if err != nil {
return i18n.WrapError(err)
}

var objectFiles []string
objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes, verbose, warningsLevel, logger)
objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes)
if err != nil {
return i18n.WrapError(err)
}

// The "src/" subdirectory of a sketch is compiled recursively
sketchSrcPath := filepath.Join(sketchBuildPath, constants.SKETCH_FOLDER_SRC)
if info, err := os.Stat(sketchSrcPath); err == nil && info.IsDir() {
objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes, verbose, warningsLevel, logger)
objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes)
if err != nil {
return i18n.WrapError(err)
}
3 changes: 2 additions & 1 deletion read_file_and_store_in_context.go
Original file line number Diff line number Diff line change
@@ -36,11 +36,12 @@ import (
)

type ReadFileAndStoreInContext struct {
FileToRead string
Target *string
}

func (s *ReadFileAndStoreInContext) Run(ctx *types.Context) error {
bytes, err := ioutil.ReadFile(ctx.FileToRead)
bytes, err := ioutil.ReadFile(s.FileToRead)
if err != nil {
return i18n.WrapError(err)
}
5 changes: 2 additions & 3 deletions recipe_runner.go
Original file line number Diff line number Diff line change
@@ -34,6 +34,7 @@ import (
"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/i18n"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"os"
"sort"
"strings"
@@ -51,16 +52,14 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error {
}

buildProperties := ctx.BuildProperties.Clone()
verbose := ctx.Verbose

recipes := findRecipes(buildProperties, s.Prefix, s.Suffix)

properties := buildProperties.Clone()
for _, recipe := range recipes {
if ctx.DebugLevel >= 10 {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, constants.MSG_RUNNING_RECIPE, recipe)
}
_, err := builder_utils.ExecRecipe(properties, recipe, false, verbose, verbose, logger)
_, _, err := builder_utils.ExecRecipe(ctx, properties, recipe, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show)
if err != nil {
return i18n.WrapError(err)
}
29 changes: 22 additions & 7 deletions test/builder_utils_test.go
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ package test

import (
"github.com/arduino/arduino-builder/builder_utils"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"github.com/stretchr/testify/require"
"io/ioutil"
@@ -52,27 +53,33 @@ func tempFile(t *testing.T, prefix string) string {
}

func TestObjFileIsUpToDateObjMissing(t *testing.T) {
ctx := &types.Context{}

sourceFile := tempFile(t, "source")
defer os.RemoveAll(sourceFile)

upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, "", "")
upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, "", "")
NoError(t, err)
require.False(t, upToDate)
}

func TestObjFileIsUpToDateDepMissing(t *testing.T) {
ctx := &types.Context{}

sourceFile := tempFile(t, "source")
defer os.RemoveAll(sourceFile)

objFile := tempFile(t, "obj")
defer os.RemoveAll(objFile)

upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, "")
upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, "")
NoError(t, err)
require.False(t, upToDate)
}

func TestObjFileIsUpToDateObjOlder(t *testing.T) {
ctx := &types.Context{}

objFile := tempFile(t, "obj")
defer os.RemoveAll(objFile)
depFile := tempFile(t, "dep")
@@ -83,12 +90,14 @@ func TestObjFileIsUpToDateObjOlder(t *testing.T) {
sourceFile := tempFile(t, "source")
defer os.RemoveAll(sourceFile)

upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile)
upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile)
NoError(t, err)
require.False(t, upToDate)
}

func TestObjFileIsUpToDateObjNewer(t *testing.T) {
ctx := &types.Context{}

sourceFile := tempFile(t, "source")
defer os.RemoveAll(sourceFile)

@@ -99,12 +108,14 @@ func TestObjFileIsUpToDateObjNewer(t *testing.T) {
depFile := tempFile(t, "dep")
defer os.RemoveAll(depFile)

upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile)
upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile)
NoError(t, err)
require.True(t, upToDate)
}

func TestObjFileIsUpToDateDepIsNewer(t *testing.T) {
ctx := &types.Context{}

sourceFile := tempFile(t, "source")
defer os.RemoveAll(sourceFile)

@@ -122,12 +133,14 @@ func TestObjFileIsUpToDateDepIsNewer(t *testing.T) {

utils.WriteFile(depFile, objFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile)

upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile)
upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile)
NoError(t, err)
require.False(t, upToDate)
}

func TestObjFileIsUpToDateDepIsOlder(t *testing.T) {
ctx := &types.Context{}

sourceFile := tempFile(t, "source")
defer os.RemoveAll(sourceFile)

@@ -143,12 +156,14 @@ func TestObjFileIsUpToDateDepIsOlder(t *testing.T) {

utils.WriteFile(depFile, objFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile)

upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile)
upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile)
NoError(t, err)
require.True(t, upToDate)
}

func TestObjFileIsUpToDateDepIsWrong(t *testing.T) {
ctx := &types.Context{}

sourceFile := tempFile(t, "source")
defer os.RemoveAll(sourceFile)

@@ -166,7 +181,7 @@ func TestObjFileIsUpToDateDepIsWrong(t *testing.T) {

utils.WriteFile(depFile, sourceFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile)

upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile)
upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile)
NoError(t, err)
require.False(t, upToDate)
}
50 changes: 12 additions & 38 deletions test/includes_finder_with_regexp_test.go
Original file line number Diff line number Diff line change
@@ -43,27 +43,17 @@ func TestIncludesFinderWithRegExp(t *testing.T) {
"#include <SPI.h>\n" +
"^\n" +
"compilation terminated."
ctx.Source = output
include := builder.IncludesFinderWithRegExp(ctx, output)

parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source}
err := parser.Run(ctx)
NoError(t, err)

require.Equal(t, "SPI.h", ctx.IncludeJustFound)
require.Equal(t, "SPI.h", include)
}

func TestIncludesFinderWithRegExpEmptyOutput(t *testing.T) {
ctx := &types.Context{}

output := ""

ctx.Source = output
include := builder.IncludesFinderWithRegExp(ctx, "")

parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source}
err := parser.Run(ctx)
NoError(t, err)

require.Equal(t, "", ctx.IncludeJustFound)
require.Equal(t, "", include)
}

func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) {
@@ -73,13 +63,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) {
" # include <Wire.h>\n" +
" ^\n" +
"compilation terminated.\n"
ctx.Source = output

parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source}
err := parser.Run(ctx)
NoError(t, err)
include := builder.IncludesFinderWithRegExp(ctx, output)

require.Equal(t, "Wire.h", ctx.IncludeJustFound)
require.Equal(t, "Wire.h", include)
}

func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) {
@@ -89,13 +75,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) {
" #\t\t\tinclude <Wire.h>\n" +
" ^\n" +
"compilation terminated.\n"
ctx.Source = output
include := builder.IncludesFinderWithRegExp(ctx, output)

parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source}
err := parser.Run(ctx)
NoError(t, err)

require.Equal(t, "Wire.h", ctx.IncludeJustFound)
require.Equal(t, "Wire.h", include)
}

func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) {
@@ -104,13 +86,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) {
output := "/some/path/sketch.ino:1:33: fatal error: SPI.h: No such file or directory\n" +
"compilation terminated.\n"

ctx.Source = output

parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source}
err := parser.Run(ctx)
NoError(t, err)
include := builder.IncludesFinderWithRegExp(ctx, output)

require.Equal(t, "SPI.h", ctx.IncludeJustFound)
require.Equal(t, "SPI.h", include)
}

func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) {
@@ -119,11 +97,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) {
output := "In file included from /tmp/arduino_modified_sketch_815412/binouts.ino:52:0:\n" +
"/tmp/arduino_build_static/sketch/regtable.h:31:22: fatal error: register.h: No such file or directory\n"

ctx.Source = output

parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source}
err := parser.Run(ctx)
NoError(t, err)
include := builder.IncludesFinderWithRegExp(ctx, output)

require.Equal(t, "register.h", ctx.IncludeJustFound)
require.Equal(t, "register.h", include)
}
3 changes: 1 addition & 2 deletions test/read_file_and_store_in_context_test.go
Original file line number Diff line number Diff line change
@@ -47,9 +47,8 @@ func TestReadFileAndStoreInContext(t *testing.T) {
utils.WriteFile(file.Name(), "test test\nciao")

ctx := &types.Context{}
ctx.FileToRead = file.Name()

command := &builder.ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE}
command := &builder.ReadFileAndStoreInContext{FileToRead: file.Name(), Target: &ctx.SourceGccMinusE}
err = command.Run(ctx)
NoError(t, err)

19 changes: 19 additions & 0 deletions test/utils_test.go
Original file line number Diff line number Diff line change
@@ -70,6 +70,25 @@ func TestCommandLineParser(t *testing.T) {
require.Equal(t, "/tmp/sketch321469072.cpp", parts[22])
}

func TestPrintableCommand(t *testing.T) {
parts := []string{
"/path/to/dir with spaces/cmd",
"arg1",
"arg-\"with\"-quotes",
"specialchar-`~!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?-argument",
"arg with spaces",
"arg\twith\t\ttabs",
"lastarg",
}
correct := "\"/path/to/dir with spaces/cmd\"" +
" arg1 \"arg-\\\"with\\\"-quotes\"" +
" \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" +
" \"arg with spaces\" \"arg\twith\t\ttabs\"" +
" lastarg"
result := utils.PrintableCommand(parts)
require.Equal(t, correct, result)
}

func TestCommandLineParserError(t *testing.T) {
command := "\"command missing quote"

4 changes: 0 additions & 4 deletions types/context.go
Original file line number Diff line number Diff line change
@@ -61,7 +61,6 @@ type Context struct {
HeaderToLibraries map[string][]*Library
ImportedLibraries []*Library
LibrariesResolutionResults map[string]LibraryResolutionResult
IncludeJustFound string
IncludeFolders []string
OutputGccMinusM string

@@ -85,9 +84,6 @@ type Context struct {
// Logging
logger i18n.Logger
DebugLevel int

// ReadFileAndStoreInContext command
FileToRead string
}

func (ctx *Context) ExtractBuildOptions() properties.Map {
64 changes: 64 additions & 0 deletions utils/utils.go
Original file line number Diff line number Diff line change
@@ -30,6 +30,8 @@
package utils

import (
"fmt"
"bytes"
"crypto/md5"
"encoding/hex"
"io/ioutil"
@@ -254,6 +256,68 @@ func PrepareCommand(pattern string, logger i18n.Logger) (*exec.Cmd, error) {
return PrepareCommandFilteredArgs(pattern, filterEmptyArg, logger)
}

func printableArgument(arg string) string {
if strings.ContainsAny(arg, "\"\\ \t") {
arg = strings.Replace(arg, "\\", "\\\\", -1)
arg = strings.Replace(arg, "\"", "\\\"", -1)
return "\"" + arg + "\""
} else {
return arg
}
}

// Convert a command and argument slice back to a printable string.
// This adds basic escaping which is sufficient for debug output, but
// probably not for shell interpretation. This essentially reverses
// ParseCommandLine.
func PrintableCommand(parts []string) string {
return strings.Join(Map(parts, printableArgument), " ")
}

const (
Ignore = 0 // Redirect to null
Show = 1 // Show on stdout/stderr as normal
ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise
Capture = 3 // Capture into buffer
)

func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) {
if ctx.Verbose {
fmt.Println(PrintableCommand(command.Args))
}

if stdout == Capture {
buffer := &bytes.Buffer{}
command.Stdout = buffer
} else if stdout == Show || stdout == ShowIfVerbose && ctx.Verbose {
command.Stdout = os.Stdout
}

if stderr == Capture {
buffer := &bytes.Buffer{}
command.Stderr = buffer
} else if stderr == Show || stderr == ShowIfVerbose && ctx.Verbose {
command.Stderr = os.Stderr
}

err := command.Start()
if err != nil {
return nil, nil, i18n.WrapError(err)
}

err = command.Wait()

var outbytes, errbytes []byte
if buf, ok := command.Stdout.(*bytes.Buffer); ok {
outbytes = buf.Bytes()
}
if buf, ok := command.Stderr.(*bytes.Buffer); ok {
errbytes = buf.Bytes()
}

return outbytes, errbytes, i18n.WrapError(err)
}

func MapHas(aMap map[string]interface{}, key string) bool {
_, ok := aMap[key]
return ok