Add board search command and gRPC interface function (#1210)

* Add board search command

* board search now searches only on board name and results are sorted alphabetically

* Remove fuzzy search from board search command

Author: silvanocerza

arduino/utils/search.go

@@ -0,0 +1,66 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package utils
+
+import (
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/runes"
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
+)
+
+// removeDiatrics removes accents and similar diatrics from unicode characters.
+// An empty string is returned in case of errors.
+// This might not be the best solution but it works well enough for our usecase,
+// in the future we might want to use the golang.org/x/text/secure/precis package
+// when its API will be finalized.
+// From https://stackoverflow.com/a/26722698
+func removeDiatrics(s string) (string, error) {
+	transformer := transform.Chain(
+		norm.NFD,
+		runes.Remove(runes.In(unicode.Mn)),
+		norm.NFC,
+	)
+	s, _, err := transform.String(transformer, s)
+	if err != nil {
+		return "", err
+	}
+	return s, nil
+}
+
+// Match returns true if all substrings are contained in str.
+// Both str and substrings are transforms to lower case and have their
+// accents and other unicode diatrics removed.
+// If strings transformation fails an error is returned.
+func Match(str string, substrings []string) (bool, error) {
+	str, err := removeDiatrics(strings.ToLower(str))
+	if err != nil {
+		return false, err
+	}
+
+	for _, sub := range substrings {
+		cleanSub, err := removeDiatrics(strings.ToLower(sub))
+		if err != nil {
+			return false, err
+		}
+		if !strings.Contains(str, cleanSub) {
+			return false, nil
+		}
+	}
+	return true, nil
+}

cli/board/board.go

@@ -37,6 +37,7 @@ func NewCommand() *cobra.Command {
 	boardCommand.AddCommand(initDetailsCommand())
 	boardCommand.AddCommand(initListCommand())
 	boardCommand.AddCommand(initListAllCommand())
+	boardCommand.AddCommand(initSearchCommand())
 
 	return boardCommand
 }

cli/board/search.go

@@ -0,0 +1,99 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package board
+
+import (
+	"context"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/arduino/arduino-cli/cli/errorcodes"
+	"github.com/arduino/arduino-cli/cli/feedback"
+	"github.com/arduino/arduino-cli/cli/instance"
+	"github.com/arduino/arduino-cli/commands/board"
+	rpc "github.com/arduino/arduino-cli/rpc/commands"
+	"github.com/arduino/arduino-cli/table"
+	"github.com/spf13/cobra"
+)
+
+func initSearchCommand() *cobra.Command {
+	var searchCommand = &cobra.Command{
+		Use:   "search [boardname]",
+		Short: "List all known boards and their corresponding FQBN.",
+		Long: "" +
+			"List all boards that have the support platform installed. You can search\n" +
+			"for a specific board if you specify the board name",
+		Example: "" +
+			"  " + os.Args[0] + " board search\n" +
+			"  " + os.Args[0] + " board search zero",
+		Args: cobra.ArbitraryArgs,
+		Run:  runSearchCommand,
+	}
+	searchCommand.Flags().BoolVarP(&searchFlags.showHiddenBoard, "show-hidden", "a", false, "Show also boards marked as 'hidden' in the platform")
+	return searchCommand
+}
+
+var searchFlags struct {
+	showHiddenBoard bool
+}
+
+func runSearchCommand(cmd *cobra.Command, args []string) {
+	inst, err := instance.CreateInstance()
+	if err != nil {
+		feedback.Errorf("Error searching boards: %v", err)
+		os.Exit(errorcodes.ErrGeneric)
+	}
+
+	res, err := board.Search(context.Background(), &rpc.BoardSearchReq{
+		Instance:            inst,
+		SearchArgs:          strings.Join(args, " "),
+		IncludeHiddenBoards: searchFlags.showHiddenBoard,
+	})
+	if err != nil {
+		feedback.Errorf("Error searching boards: %v", err)
+		os.Exit(errorcodes.ErrGeneric)
+	}
+
+	feedback.PrintResult(searchResults{res.Boards})
+}
+
+// output from this command requires special formatting so we create a dedicated
+// feedback.Result implementation
+type searchResults struct {
+	boards []*rpc.BoardListItem
+}
+
+func (r searchResults) Data() interface{} {
+	return r.boards
+}
+
+func (r searchResults) String() string {
+	sort.Slice(r.boards, func(i, j int) bool {
+		return r.boards[i].GetName() < r.boards[j].GetName()
+	})
+
+	t := table.New()
+	t.SetHeader("Board Name", "FQBN", "Platform ID", "")
+	for _, item := range r.boards {
+		hidden := ""
+		if item.IsHidden {
+			hidden = "(hidden)"
+		}
+		t.AddRow(item.GetName(), item.GetFQBN(), item.Platform.ID, hidden)
+	}
+	return t.Render()
+}

client_example/go.sum

@@ -150,6 +150,9 @@ func main() {
 	log.Println("calling BoardDetails(arduino:samd:mkr1000)")
 	callBoardsDetails(client, instance)
 
+	log.Println("calling BoardSearch()")
+	callBoardSearch(client, instance)
+
 	// Attach a board to a sketch.
 	// Uncomment if you do have an actual board connected.
 	// log.Println("calling BoardAttach(serial:///dev/ttyACM0)")
@@ -524,6 +527,22 @@ func callBoardsDetails(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
 	log.Printf("Config options: %s", details.GetConfigOptions())
 }
 
+func callBoardSearch(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
+	res, err := client.BoardSearch(context.Background(),
+		&rpc.BoardSearchReq{
+			Instance:   instance,
+			SearchArgs: "",
+		})
+
+	if err != nil {
+		log.Fatalf("Error getting board data: %s\n", err)
+	}
+
+	for _, board := range res.Boards {
+		log.Printf("Board Name: %s, Board Platform: %s\n", board.Name, board.Platform.ID)
+	}
+}
+
 func callBoardAttach(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
 	currDir, _ := os.Getwd()
 	boardattachresp, err := client.BoardAttach(context.Background(),

client_example/main.go

@@ -0,0 +1,128 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package board
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino/utils"
+	"github.com/arduino/arduino-cli/commands"
+	rpc "github.com/arduino/arduino-cli/rpc/commands"
+)
+
+// Search returns all boards that match the search arg.
+// Boards are searched in all platforms, including those in the index that are not yet
+// installed. Note that platforms that are not installed don't include boards' FQBNs.
+// If no search argument is used all boards are returned.
+func Search(ctx context.Context, req *rpc.BoardSearchReq) (*rpc.BoardSearchResp, error) {
+	pm := commands.GetPackageManager(req.GetInstance().GetId())
+	if pm == nil {
+		return nil, errors.New("invalid instance")
+	}
+
+	searchArgs := strings.Split(strings.Trim(req.SearchArgs, " "), " ")
+
+	match := func(toTest []string) (bool, error) {
+		if len(searchArgs) == 0 {
+			return true, nil
+		}
+
+		for _, t := range toTest {
+			matches, err := utils.Match(t, searchArgs)
+			if err != nil {
+				return false, err
+			}
+			if matches {
+				return matches, nil
+			}
+		}
+		return false, nil
+	}
+
+	res := &rpc.BoardSearchResp{Boards: []*rpc.BoardListItem{}}
+	for _, targetPackage := range pm.Packages {
+		for _, platform := range targetPackage.Platforms {
+			latestPlatformRelease := platform.GetLatestRelease()
+			if latestPlatformRelease == nil {
+				continue
+			}
+			installedVersion := ""
+			if installedPlatformRelease := pm.GetInstalledPlatformRelease(platform); installedPlatformRelease != nil {
+				installedVersion = installedPlatformRelease.Version.String()
+			}
+
+			rpcPlatform := &rpc.Platform{
+				ID:                platform.String(),
+				Installed:         installedVersion,
+				Latest:            latestPlatformRelease.Version.String(),
+				Name:              platform.Name,
+				Maintainer:        platform.Package.Maintainer,
+				Website:           platform.Package.WebsiteURL,
+				Email:             platform.Package.Email,
+				ManuallyInstalled: platform.ManuallyInstalled,
+			}
+
+			// Platforms that are not installed don't have a list of boards
+			// generated from their boards.txt file so we need two different
+			// ways of reading board data.
+			// The only boards information for platforms that are not installed
+			// is that found in the index, usually that's only a board name.
+			if len(latestPlatformRelease.Boards) != 0 {
+				for _, board := range latestPlatformRelease.Boards {
+					if !req.GetIncludeHiddenBoards() && board.IsHidden() {
+						continue
+					}
+
+					toTest := append(strings.Split(board.Name(), " "), board.Name(), board.FQBN())
+					if ok, err := match(toTest); err != nil {
+						return nil, err
+					} else if !ok {
+						continue
+					}
+
+					res.Boards = append(res.Boards, &rpc.BoardListItem{
+						Name:     board.Name(),
+						FQBN:     board.FQBN(),
+						IsHidden: board.IsHidden(),
+						Platform: rpcPlatform,
+					})
+				}
+			} else {
+				for _, board := range latestPlatformRelease.BoardsManifest {
+					toTest := append(strings.Split(board.Name, " "), board.Name)
+					if ok, err := match(toTest); err != nil {
+						return nil, err
+					} else if !ok {
+						continue
+					}
+
+					res.Boards = append(res.Boards, &rpc.BoardListItem{
+						Name:     strings.Trim(board.Name, " \n"),
+						Platform: rpcPlatform,
+					})
+				}
+			}
+		}
+	}
+
+	sort.Slice(res.Boards, func(i, j int) bool {
+		return res.Boards[i].Name < res.Boards[j].Name
+	})
+	return res, nil
+}

commands/board/search.go

@@ -59,6 +59,11 @@ func (s *ArduinoCoreServerImpl) BoardListAll(ctx context.Context, req *rpc.Board
 	return board.ListAll(ctx, req)
 }
 
+// BoardSearch exposes to the gRPC interface the board search command
+func (s *ArduinoCoreServerImpl) BoardSearch(ctx context.Context, req *rpc.BoardSearchReq) (*rpc.BoardSearchResp, error) {
+	return board.Search(ctx, req)
+}
+
 // BoardListWatch FIXMEDOC
 func (s *ArduinoCoreServerImpl) BoardListWatch(stream rpc.ArduinoCore_BoardListWatchServer) error {
 	msg, err := stream.Recv()

commands/daemon/daemon.go

@@ -238,3 +238,17 @@ message BoardListItem {
     // Platform this board belongs to
     Platform platform = 6;
 }
+
+message BoardSearchReq {
+    // Arduino Core Service instance from the `Init` response.
+    Instance instance = 1;
+    // The search query to filter the board list by.
+    string search_args = 2;
+    // Set to true to get also the boards marked as "hidden" in installed platforms
+    bool include_hidden_boards = 3;
+}
+
+message BoardSearchResp {
+    // List of installed and installable boards.
+    repeated BoardListItem boards = 1;
+}

rpc/commands/board.pb.go

@@ -80,6 +80,9 @@ service ArduinoCore {
   // List all the boards provided by installed platforms.
   rpc BoardListAll(BoardListAllReq) returns (BoardListAllResp);
 
+  // Search boards in installed and not installed Platforms.
+  rpc BoardSearch(BoardSearchReq) returns (BoardSearchResp);
+
   // List boards connection and disconnected events.
   rpc BoardListWatch(stream BoardListWatchReq) returns (stream BoardListWatchResp);
 

rpc/commands/board.proto

@@ -587,3 +587,91 @@ def test_board_details_list_programmers_flag(run_command):
     assert "edbg      Atmel EDBG" in lines
     assert "atmel_ice Atmel-ICE" in lines
     assert "sam_ice   Atmel SAM-ICE" in lines
+
+
+def test_board_search(run_command, data_dir):
+    assert run_command("update")
+
+    res = run_command("board search --format json")
+    assert res.ok
+    data = json.loads(res.stdout)
+    # Verifies boards are returned
+    assert len(data) > 0
+    # Verifies no board has FQBN set since no platform is installed
+    assert len([board["FQBN"] for board in data if "FQBN" in board]) == 0
+    names = [board["name"] for board in data if "name" in board]
+    assert "Arduino Uno" in names
+    assert "Arduino Yún" in names
+    assert "Arduino Zero" in names
+    assert "Arduino Nano 33 BLE" in names
+    assert "Arduino Portenta H7" in names
+
+    # Search in non installed boards
+    res = run_command("board search --format json nano 33")
+    assert res.ok
+    data = json.loads(res.stdout)
+    # Verifies boards are returned
+    assert len(data) > 0
+    # Verifies no board has FQBN set since no platform is installed
+    assert len([board["FQBN"] for board in data if "FQBN" in board]) == 0
+    names = [board["name"] for board in data if "name" in board]
+    assert "Arduino Nano 33 BLE" in names
+    assert "Arduino Nano 33 IoT" in names
+
+    # Install a platform from index
+    assert run_command("core install arduino:avr@1.8.3")
+
+    res = run_command("board search --format json")
+    assert res.ok
+    data = json.loads(res.stdout)
+    assert len(data) > 0
+    # Verifies some FQBNs are now returned after installing a platform
+    assert len([board["FQBN"] for board in data if "FQBN" in board]) == 26
+    installed_boards = {board["FQBN"]: board for board in data if "FQBN" in board}
+    assert "arduino:avr:uno" in installed_boards
+    assert "Arduino Uno" == installed_boards["arduino:avr:uno"]["name"]
+    assert "arduino:avr:yun" in installed_boards
+    assert "Arduino Yún" == installed_boards["arduino:avr:yun"]["name"]
+
+    res = run_command("board search --format json arduino yun")
+    assert res.ok
+    data = json.loads(res.stdout)
+    assert len(data) > 0
+    installed_boards = {board["FQBN"]: board for board in data if "FQBN" in board}
+    assert "arduino:avr:yun" in installed_boards
+    assert "Arduino Yún" == installed_boards["arduino:avr:yun"]["name"]
+
+    # Manually installs a core in sketchbooks hardware folder
+    git_url = "https://github.com/arduino/ArduinoCore-samd.git"
+    repo_dir = Path(data_dir, "hardware", "arduino-beta-development", "samd")
+    assert Repo.clone_from(git_url, repo_dir, multi_options=["-b 1.8.11"])
+
+    res = run_command("board search --format json")
+    assert res.ok
+    data = json.loads(res.stdout)
+    assert len(data) > 0
+    # Verifies some FQBNs are now returned after installing a platform
+    assert len([board["FQBN"] for board in data if "FQBN" in board]) == 43
+    installed_boards = {board["FQBN"]: board for board in data if "FQBN" in board}
+    assert "arduino:avr:uno" in installed_boards
+    assert "Arduino Uno" == installed_boards["arduino:avr:uno"]["name"]
+    assert "arduino:avr:yun" in installed_boards
+    assert "Arduino Yún" == installed_boards["arduino:avr:yun"]["name"]
+    assert "arduino-beta-development:samd:mkrwifi1010" in installed_boards
+    assert "Arduino MKR WiFi 1010" == installed_boards["arduino-beta-development:samd:mkrwifi1010"]["name"]
+    assert "arduino-beta-development:samd:mkr1000" in installed_boards
+    assert "Arduino MKR1000" == installed_boards["arduino-beta-development:samd:mkr1000"]["name"]
+    assert "arduino-beta-development:samd:mkrzero" in installed_boards
+    assert "Arduino MKRZERO" == installed_boards["arduino-beta-development:samd:mkrzero"]["name"]
+    assert "arduino-beta-development:samd:nano_33_iot" in installed_boards
+    assert "Arduino NANO 33 IoT" == installed_boards["arduino-beta-development:samd:nano_33_iot"]["name"]
+    assert "arduino-beta-development:samd:arduino_zero_native" in installed_boards
+
+    res = run_command("board search --format json mkr1000")
+    assert res.ok
+    data = json.loads(res.stdout)
+    assert len(data) > 0
+    # Verifies some FQBNs are now returned after installing a platform
+    installed_boards = {board["FQBN"]: board for board in data if "FQBN" in board}
+    assert "arduino-beta-development:samd:mkr1000" in installed_boards
+    assert "Arduino MKR1000" == installed_boards["arduino-beta-development:samd:mkr1000"]["name"]