Skip to content
/ arduino-create-agent Public

Commit 04414e2

Browse files
MatteoPologrutoMatteoPologruto
authoredJul 2, 2024··
Check if a signed URL is specified and use it download tools (#953)
* Wrap v2 tools install function inside tools.Download * Download tools defaulting to the replace behaviour * Improve archive renamer and fix failing tests * Find the correct tool and system when `version=latest` is specified * Reintroduce caching option when downloading tools
·
1.7.1-rc11.6.0-rc1
1 parent b02967e commit 04414e2

File tree

6 files changed

+142
-211
lines changed

6 files changed

+142
-211
lines changed
 

‎tools/download.go

Lines changed: 13 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -16,43 +16,18 @@
1616
package tools
1717

1818
import (
19-
"bytes"
2019
"context"
21-
"crypto/sha256"
22-
"encoding/hex"
23-
"encoding/json"
2420
"errors"
25-
"fmt"
26-
"io"
27-
"net/http"
2821
"os"
2922
"os/exec"
3023
"path/filepath"
3124
"runtime"
3225

26+
"github.com/arduino/arduino-create-agent/gen/tools"
27+
"github.com/arduino/arduino-create-agent/utilities"
3328
"github.com/arduino/arduino-create-agent/v2/pkgs"
34-
"github.com/arduino/go-paths-helper"
35-
"github.com/blang/semver"
36-
"github.com/codeclysm/extract/v3"
3729
)
3830

39-
// public vars to allow override in the tests
40-
var (
41-
OS = runtime.GOOS
42-
Arch = runtime.GOARCH
43-
)
44-
45-
func pathExists(path string) bool {
46-
_, err := os.Stat(path)
47-
if err == nil {
48-
return true
49-
}
50-
if os.IsNotExist(err) {
51-
return false
52-
}
53-
return true
54-
}
55-
5631
// Download will parse the index at the indexURL for the tool to download.
5732
// It will extract it in a folder in .arduino-create, and it will update the
5833
// Installed map.
@@ -70,97 +45,21 @@ func pathExists(path string) bool {
7045
// if it already exists.
7146
func (t *Tools) Download(pack, name, version, behaviour string) error {
7247

73-
body, err := t.index.Read()
74-
if err != nil {
75-
return err
76-
}
77-
78-
var data pkgs.Index
79-
json.Unmarshal(body, &data)
80-
81-
// Find the tool by name
82-
correctTool, correctSystem := findTool(pack, name, version, data)
83-
84-
if correctTool.Name == "" || correctSystem.URL == "" {
85-
t.logger("We couldn't find a tool with the name " + name + " and version " + version + " packaged by " + pack)
86-
return nil
87-
}
88-
89-
key := correctTool.Name + "-" + correctTool.Version
90-
91-
// Check if it already exists
92-
if behaviour == "keep" {
93-
location, ok := t.getMapValue(key)
94-
if ok && pathExists(location) {
95-
// overwrite the default tool with this one
96-
t.setMapValue(correctTool.Name, location)
97-
t.logger("The tool is already present on the system")
98-
return t.writeMap()
99-
}
100-
}
101-
102-
// Download the tool
103-
t.logger("Downloading tool " + name + " from " + correctSystem.URL)
104-
resp, err := http.Get(correctSystem.URL)
48+
tool := pkgs.New(t.index, t.directory.String(), behaviour)
49+
_, err := tool.Install(context.Background(), &tools.ToolPayload{Name: name, Version: version, Packager: pack})
10550
if err != nil {
10651
return err
10752
}
108-
defer resp.Body.Close()
109-
110-
// Read the body
111-
body, err = io.ReadAll(resp.Body)
112-
if err != nil {
113-
return err
114-
}
115-
116-
// Checksum
117-
checksum := sha256.Sum256(body)
118-
checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size])
119-
120-
if checkSumString != correctSystem.Checksum {
121-
return errors.New("checksum doesn't match")
122-
}
123-
124-
tempPath := paths.TempDir()
125-
// Create a temporary dir to extract package
126-
if err := tempPath.MkdirAll(); err != nil {
127-
return fmt.Errorf("creating temp dir for extraction: %s", err)
128-
}
129-
tempDir, err := tempPath.MkTempDir("package-")
130-
if err != nil {
131-
return fmt.Errorf("creating temp dir for extraction: %s", err)
132-
}
133-
defer tempDir.RemoveAll()
13453

135-
t.logger("Unpacking tool " + name)
136-
ctx := context.Background()
137-
reader := bytes.NewReader(body)
138-
// Extract into temp directory
139-
if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil {
140-
return fmt.Errorf("extracting archive: %s", err)
141-
}
142-
143-
location := t.directory.Join(pack, correctTool.Name, correctTool.Version)
144-
err = location.RemoveAll()
54+
path := filepath.Join(pack, name, version)
55+
safePath, err := utilities.SafeJoin(t.directory.String(), path)
14556
if err != nil {
14657
return err
14758
}
14859

149-
// Check package content and find package root dir
150-
root, err := findPackageRoot(tempDir)
151-
if err != nil {
152-
return fmt.Errorf("searching package root dir: %s", err)
153-
}
154-
155-
if err := root.Rename(location); err != nil {
156-
if err := root.CopyDirTo(location); err != nil {
157-
return fmt.Errorf("moving extracted archive to destination dir: %s", err)
158-
}
159-
}
160-
16160
// if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers
16261
// AFAIK this is only the case for the windows-driver tool
163-
err = t.installDrivers(location.String())
62+
err = t.installDrivers(safePath)
16463
if err != nil {
16564
return err
16665
}
@@ -169,63 +68,20 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
16968
t.logger("Ensure that the files are executable")
17069

17170
// Update the tool map
172-
t.logger("Updating map with location " + location.String())
173-
174-
t.setMapValue(name, location.String())
175-
t.setMapValue(name+"-"+correctTool.Version, location.String())
176-
return t.writeMap()
177-
}
71+
t.logger("Updating map with location " + safePath)
17872

179-
func findPackageRoot(parent *paths.Path) (*paths.Path, error) {
180-
files, err := parent.ReadDir()
181-
if err != nil {
182-
return nil, fmt.Errorf("reading package root dir: %s", err)
183-
}
184-
files.FilterOutPrefix("__MACOSX")
73+
t.setMapValue(name, safePath)
74+
t.setMapValue(name+"-"+version, safePath)
18575

186-
// if there is only one dir, it is the root dir
187-
if len(files) == 1 && files[0].IsDir() {
188-
return files[0], nil
189-
}
190-
return parent, nil
191-
}
192-
193-
func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) {
194-
var correctTool pkgs.Tool
195-
correctTool.Version = "0.0"
196-
197-
for _, p := range data.Packages {
198-
if p.Name != pack {
199-
continue
200-
}
201-
for _, t := range p.Tools {
202-
if version != "latest" {
203-
if t.Name == name && t.Version == version {
204-
correctTool = t
205-
}
206-
} else {
207-
// Find latest
208-
v1, _ := semver.Make(t.Version)
209-
v2, _ := semver.Make(correctTool.Version)
210-
if t.Name == name && v1.Compare(v2) > 0 {
211-
correctTool = t
212-
}
213-
}
214-
}
215-
}
216-
217-
// Find the url based on system
218-
correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch)
219-
220-
return correctTool, correctSystem
76+
return nil
22177
}
22278

22379
func (t *Tools) installDrivers(location string) error {
22480
OkPressed := 6
22581
extension := ".bat"
22682
// add .\ to force locality
22783
preamble := ".\\"
228-
if OS != "windows" {
84+
if runtime.GOOS != "windows" {
22985
extension = ".sh"
23086
// add ./ to force locality
23187
preamble = "./"
@@ -237,7 +93,7 @@ func (t *Tools) installDrivers(location string) error {
23793
os.Chdir(location)
23894
t.logger(preamble + "post_install" + extension)
23995
oscmd := exec.Command(preamble + "post_install" + extension)
240-
if OS != "linux" {
96+
if runtime.GOOS != "linux" {
24197
// spawning a shell could be the only way to let the user type his password
24298
TellCommandNotToSpawnShell(oscmd)
24399
}

‎tools/download_test.go

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ func TestDownloadCorrectPlatform(t *testing.T) {
4242
{"linux", "arm", "arm-linux-gnueabihf"},
4343
}
4444
defer func() {
45-
OS = runtime.GOOS // restore `runtime.OS`
46-
Arch = runtime.GOARCH // restore `runtime.ARCH`
45+
pkgs.OS = runtime.GOOS // restore `runtime.OS`
46+
pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
4747
}()
4848
testIndex := paths.New("testdata", "test_tool_index.json")
4949
buf, err := testIndex.ReadFile()
@@ -54,10 +54,11 @@ func TestDownloadCorrectPlatform(t *testing.T) {
5454
require.NoError(t, err)
5555
for _, tc := range testCases {
5656
t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
57-
OS = tc.hostOS // override `runtime.OS` for testing purposes
58-
Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
57+
pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes
58+
pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
5959
// Find the tool by name
60-
correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
60+
correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
61+
require.True(t, found)
6162
require.NotNil(t, correctTool)
6263
require.NotNil(t, correctSystem)
6364
require.Equal(t, correctTool.Name, "arduino-fwuploader")
@@ -78,8 +79,8 @@ func TestDownloadFallbackPlatform(t *testing.T) {
7879
{"windows", "amd64", "i686-mingw32"},
7980
}
8081
defer func() {
81-
OS = runtime.GOOS // restore `runtime.OS`
82-
Arch = runtime.GOARCH // restore `runtime.ARCH`
82+
pkgs.OS = runtime.GOOS // restore `runtime.OS`
83+
pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
8384
}()
8485
testIndex := paths.New("testdata", "test_tool_index.json")
8586
buf, err := testIndex.ReadFile()
@@ -90,10 +91,11 @@ func TestDownloadFallbackPlatform(t *testing.T) {
9091
require.NoError(t, err)
9192
for _, tc := range testCases {
9293
t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
93-
OS = tc.hostOS // override `runtime.OS` for testing purposes
94-
Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
94+
pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes
95+
pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
9596
// Find the tool by name
96-
correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
97+
correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
98+
require.True(t, found)
9799
require.NotNil(t, correctTool)
98100
require.NotNil(t, correctSystem)
99101
require.Equal(t, correctTool.Name, "arduino-fwuploader")
@@ -145,7 +147,7 @@ func TestDownload(t *testing.T) {
145147
if filePath.IsDir() {
146148
require.DirExists(t, filePath.String())
147149
} else {
148-
if OS == "windows" {
150+
if runtime.GOOS == "windows" {
149151
require.FileExists(t, filePath.String()+".exe")
150152
} else {
151153
require.FileExists(t, filePath.String())

‎tools/tools.go

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,18 +78,6 @@ func (t *Tools) getMapValue(key string) (string, bool) {
7878
return value, ok
7979
}
8080

81-
// writeMap() writes installed map to the json file "installed.json"
82-
func (t *Tools) writeMap() error {
83-
t.mutex.RLock()
84-
defer t.mutex.RUnlock()
85-
b, err := json.Marshal(t.installed)
86-
if err != nil {
87-
return err
88-
}
89-
filePath := t.directory.Join("installed.json")
90-
return filePath.WriteFile(b)
91-
}
92-
9381
// readMap() reads the installed map from json file "installed.json"
9482
func (t *Tools) readMap() error {
9583
t.mutex.Lock()

‎v2/http.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func Server(directory string, index *index.Resource) http.Handler {
4040
logAdapter := LogAdapter{Logger: logger}
4141

4242
// Mount tools
43-
toolsSvc := pkgs.New(index, directory)
43+
toolsSvc := pkgs.New(index, directory, "replace")
4444
toolsEndpoints := toolssvc.NewEndpoints(toolsSvc)
4545
toolsServer := toolssvr.New(toolsEndpoints, mux, CustomRequestDecoder, goahttp.ResponseEncoder, errorHandler(logger), nil)
4646
toolssvr.Mount(mux, toolsServer)

‎v2/pkgs/tools.go

Lines changed: 111 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,16 @@ import (
3333
"github.com/arduino/arduino-create-agent/gen/tools"
3434
"github.com/arduino/arduino-create-agent/index"
3535
"github.com/arduino/arduino-create-agent/utilities"
36+
"github.com/blang/semver"
3637
"github.com/codeclysm/extract/v3"
3738
)
3839

40+
// public vars to allow override in the tests
41+
var (
42+
OS = runtime.GOOS
43+
Arch = runtime.GOARCH
44+
)
45+
3946
// Tools is a client that implements github.com/arduino/arduino-create-agent/gen/tools.Service interface.
4047
// It saves tools in a specified folder with this structure: packager/name/version
4148
// For example:
@@ -50,17 +57,19 @@ import (
5057
//
5158
// It requires an Index Resource to search for tools
5259
type Tools struct {
53-
index *index.Resource
54-
folder string
60+
index *index.Resource
61+
folder string
62+
behaviour string
5563
}
5664

5765
// New will return a Tool object, allowing the caller to execute operations on it.
5866
// The New function will accept an index as parameter (used to download the indexes)
5967
// and a folder used to download the indexes
60-
func New(index *index.Resource, folder string) *Tools {
68+
func New(index *index.Resource, folder, behaviour string) *Tools {
6169
return &Tools{
62-
index: index,
63-
folder: folder,
70+
index: index,
71+
folder: folder,
72+
behaviour: behaviour,
6473
}
6574
}
6675

@@ -166,21 +175,28 @@ func (t *Tools) Install(ctx context.Context, payload *tools.ToolPayload) (*tools
166175
var index Index
167176
json.Unmarshal(body, &index)
168177

169-
for _, packager := range index.Packages {
170-
if packager.Name != payload.Packager {
171-
continue
172-
}
173-
174-
for _, tool := range packager.Tools {
175-
if tool.Name == payload.Name &&
176-
tool.Version == payload.Version {
177-
178-
sys := tool.GetFlavourCompatibleWith(runtime.GOOS, runtime.GOARCH)
178+
correctTool, correctSystem, found := FindTool(payload.Packager, payload.Name, payload.Version, index)
179+
path = filepath.Join(payload.Packager, correctTool.Name, correctTool.Version)
179180

180-
return t.install(ctx, path, sys.URL, sys.Checksum)
181+
key := correctTool.Name + "-" + correctTool.Version
182+
// Check if it already exists
183+
if t.behaviour == "keep" && pathExists(t.folder) {
184+
location, ok, err := checkInstalled(t.folder, key)
185+
if err != nil {
186+
return nil, err
187+
}
188+
if ok && pathExists(location) {
189+
// overwrite the default tool with this one
190+
err := writeInstalled(t.folder, path)
191+
if err != nil {
192+
return nil, err
181193
}
194+
return &tools.Operation{Status: "ok"}, nil
182195
}
183196
}
197+
if found {
198+
return t.install(ctx, path, correctSystem.URL, correctSystem.Checksum)
199+
}
184200

185201
return nil, tools.MakeNotFound(
186202
fmt.Errorf("tool not found with packager '%s', name '%s', version '%s'",
@@ -256,27 +272,51 @@ func (t *Tools) Remove(ctx context.Context, payload *tools.ToolPayload) (*tools.
256272
func rename(base string) extract.Renamer {
257273
return func(path string) string {
258274
parts := strings.Split(filepath.ToSlash(path), "/")
259-
path = strings.Join(parts[1:], "/")
260-
path = filepath.Join(base, path)
275+
newPath := strings.Join(parts[1:], "/")
276+
if newPath == "" {
277+
newPath = filepath.Join(newPath, path)
278+
}
279+
path = filepath.Join(base, newPath)
261280
return path
262281
}
263282
}
264283

265-
func writeInstalled(folder, path string) error {
284+
func readInstalled(installedFile string) (map[string]string, error) {
266285
// read installed.json
267286
installed := map[string]string{}
268-
269-
installedFile, err := utilities.SafeJoin(folder, "installed.json")
270-
if err != nil {
271-
return err
272-
}
273287
data, err := os.ReadFile(installedFile)
274288
if err == nil {
275289
err = json.Unmarshal(data, &installed)
276290
if err != nil {
277-
return err
291+
return nil, err
278292
}
279293
}
294+
return installed, nil
295+
}
296+
297+
func checkInstalled(folder, key string) (string, bool, error) {
298+
installedFile, err := utilities.SafeJoin(folder, "installed.json")
299+
if err != nil {
300+
return "", false, err
301+
}
302+
installed, err := readInstalled(installedFile)
303+
if err != nil {
304+
return "", false, err
305+
}
306+
location, ok := installed[key]
307+
return location, ok, err
308+
}
309+
310+
func writeInstalled(folder, path string) error {
311+
// read installed.json
312+
installedFile, err := utilities.SafeJoin(folder, "installed.json")
313+
if err != nil {
314+
return err
315+
}
316+
installed, err := readInstalled(installedFile)
317+
if err != nil {
318+
return err
319+
}
280320

281321
parts := strings.Split(path, string(filepath.Separator))
282322
tool := parts[len(parts)-2]
@@ -288,10 +328,55 @@ func writeInstalled(folder, path string) error {
288328
installed[tool] = toolFile
289329
installed[toolWithVersion] = toolFile
290330

291-
data, err = json.Marshal(installed)
331+
data, err := json.Marshal(installed)
292332
if err != nil {
293333
return err
294334
}
295335

296336
return os.WriteFile(installedFile, data, 0644)
297337
}
338+
339+
func pathExists(path string) bool {
340+
_, err := os.Stat(path)
341+
if err == nil {
342+
return true
343+
}
344+
if os.IsNotExist(err) {
345+
return false
346+
}
347+
return true
348+
}
349+
350+
// FindTool searches the index for the correct tool and system that match the specified tool name and version
351+
func FindTool(pack, name, version string, data Index) (Tool, System, bool) {
352+
var correctTool Tool
353+
correctTool.Version = "0.0"
354+
found := false
355+
356+
for _, p := range data.Packages {
357+
if p.Name != pack {
358+
continue
359+
}
360+
for _, t := range p.Tools {
361+
if version != "latest" {
362+
if t.Name == name && t.Version == version {
363+
correctTool = t
364+
found = true
365+
}
366+
} else {
367+
// Find latest
368+
v1, _ := semver.Make(t.Version)
369+
v2, _ := semver.Make(correctTool.Version)
370+
if t.Name == name && v1.Compare(v2) > 0 {
371+
correctTool = t
372+
found = true
373+
}
374+
}
375+
}
376+
}
377+
378+
// Find the url based on system
379+
correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch)
380+
381+
return correctTool, correctSystem, found
382+
}

‎v2/pkgs/tools_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func TestTools(t *testing.T) {
4545
// Instantiate Index
4646
Index := index.Init(indexURL, config.GetDataDir())
4747

48-
service := pkgs.New(Index, tmp)
48+
service := pkgs.New(Index, tmp, "replace")
4949

5050
ctx := context.Background()
5151

@@ -126,7 +126,7 @@ func TestEvilFilename(t *testing.T) {
126126
// Instantiate Index
127127
Index := index.Init(indexURL, config.GetDataDir())
128128

129-
service := pkgs.New(Index, tmp)
129+
service := pkgs.New(Index, tmp, "replace")
130130

131131
ctx := context.Background()
132132

@@ -195,7 +195,7 @@ func TestInstalledHead(t *testing.T) {
195195
// Instantiate Index
196196
Index := index.Init(indexURL, config.GetDataDir())
197197

198-
service := pkgs.New(Index, tmp)
198+
service := pkgs.New(Index, tmp, "replace")
199199

200200
ctx := context.Background()
201201

@@ -216,7 +216,7 @@ func TestInstall(t *testing.T) {
216216
LastRefresh: time.Now(),
217217
}
218218

219-
tool := pkgs.New(testIndex, tmp)
219+
tool := pkgs.New(testIndex, tmp, "replace")
220220

221221
ctx := context.Background()
222222

0 commit comments

Comments
 (0)
Please sign in to comment.