Skip to content

Commit dd23914

Browse files
olpawchumer
olpaw
authored and
chumer
committed
[GR-47186] Fix native-image --add-modules.
PullRequest: graal/15069
2 parents 1ee928c + e4e4dc5 commit dd23914

File tree

3 files changed

+128
-16
lines changed

3 files changed

+128
-16
lines changed

‎substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ public boolean consume(ArgumentQueue args) {
9393
if (addModulesArgs == null) {
9494
NativeImage.showError(headArg + moduleSetModifierOptionErrorMessage);
9595
}
96-
nativeImage.addImageBuilderJavaArgs(addModulesOption, addModulesArgs);
9796
nativeImage.addAddedModules(addModulesArgs);
9897
return true;
9998
case limitModulesOption:
@@ -189,7 +188,6 @@ public boolean consume(ArgumentQueue args) {
189188
if (addModulesArgs.isEmpty()) {
190189
NativeImage.showError(headArg + moduleSetModifierOptionErrorMessage);
191190
}
192-
nativeImage.addImageBuilderJavaArgs(addModulesOption, addModulesArgs);
193191
nativeImage.addAddedModules(addModulesArgs);
194192
return true;
195193
}

‎substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ private void applyEnabled(MacroOption.EnabledOption enabledOption, String argume
9595

9696
enabledOption.forEachPropertyValue(config,
9797
"ImageBuilderClasspath", entry -> nativeImage.addImageBuilderClasspath(Path.of(entry)), PATH_SEPARATOR_REGEX);
98+
enabledOption.forEachPropertyValue(config,
99+
"ImageBuilderModulePath", entry -> nativeImage.addImageBuilderModulePath(Path.of(entry)), PATH_SEPARATOR_REGEX);
98100
boolean explicitImageModulePath = enabledOption.forEachPropertyValue(config,
99101
"ImageModulePath", entry -> nativeImage.addImageModulePath(Path.of((entry))), PATH_SEPARATOR_REGEX);
100102
boolean explicitImageClasspath = enabledOption.forEachPropertyValue(config,

‎substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java

Lines changed: 126 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.io.InputStream;
3131
import java.io.InputStreamReader;
3232
import java.lang.reflect.Method;
33+
import java.net.URI;
3334
import java.nio.charset.StandardCharsets;
3435
import java.nio.file.Files;
3536
import java.nio.file.InvalidPathException;
@@ -44,10 +45,12 @@
4445
import java.util.HashMap;
4546
import java.util.HashSet;
4647
import java.util.Iterator;
48+
import java.util.LinkedHashMap;
4749
import java.util.LinkedHashSet;
4850
import java.util.List;
4951
import java.util.ListIterator;
5052
import java.util.Map;
53+
import java.util.Objects;
5154
import java.util.Optional;
5255
import java.util.Properties;
5356
import java.util.Set;
@@ -56,6 +59,7 @@
5659
import java.util.function.BiFunction;
5760
import java.util.function.Consumer;
5861
import java.util.function.Function;
62+
import java.util.function.Predicate;
5963
import java.util.jar.Attributes;
6064
import java.util.jar.JarFile;
6165
import java.util.jar.Manifest;
@@ -1178,9 +1182,6 @@ private int completeImageBuild() {
11781182
return ExitStatus.FALLBACK_IMAGE.getValue();
11791183
}
11801184

1181-
if (!addModules.isEmpty()) {
1182-
imageBuilderJavaArgs.add("-D" + ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES + "=" + String.join(",", addModules));
1183-
}
11841185
if (!limitModules.isEmpty()) {
11851186
imageBuilderJavaArgs.add("-D" + ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_LIMITED_MODULES + "=" + String.join(",", limitModules));
11861187
}
@@ -1425,20 +1426,13 @@ protected int buildImage(List<String> javaArgs, LinkedHashSet<Path> cp, LinkedHa
14251426
if (!cp.isEmpty()) {
14261427
arguments.addAll(Arrays.asList("-cp", cp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator))));
14271428
}
1429+
14281430
if (!mp.isEmpty()) {
14291431
List<String> strings = Arrays.asList("--module-path", mp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator)));
14301432
arguments.addAll(strings);
14311433
}
14321434

1433-
arguments.addAll(config.getGeneratorMainClass());
1434-
1435-
if (IS_AOT && OS.getCurrent().hasProcFS) {
1436-
/*
1437-
* GR-8254: Ensure image-building VM shuts down even if native-image dies unexpected
1438-
* (e.g. using CTRL-C in Gradle daemon mode)
1439-
*/
1440-
arguments.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID()));
1441-
}
1435+
String javaExecutable = canonicalize(config.getJavaExecutable()).toString();
14421436

14431437
if (useBundle()) {
14441438
LogUtils.warning("Native Image Bundles are an experimental feature.");
@@ -1450,12 +1444,68 @@ protected int buildImage(List<String> javaArgs, LinkedHashSet<Path> cp, LinkedHa
14501444
Function<Path, Path> substituteClassPath = useBundle() ? bundleSupport::substituteClassPath : Function.identity();
14511445
List<Path> finalImageClassPath = imagecp.stream().map(substituteClassPath).collect(Collectors.toList());
14521446
Function<Path, Path> substituteModulePath = useBundle() ? bundleSupport::substituteModulePath : Function.identity();
1453-
List<Path> finalImageModulePath = imagemp.stream().map(substituteModulePath).collect(Collectors.toList());
1447+
List<Path> substitutedImageModulePath = imagemp.stream().map(substituteModulePath).toList();
1448+
1449+
Map<String, Path> modules = listModulesFromPath(javaExecutable, Stream.concat(mp.stream(), imagemp.stream()).distinct().toList());
1450+
if (!addModules.isEmpty()) {
1451+
1452+
arguments.add("-D" + ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES + "=" +
1453+
String.join(",", addModules));
1454+
1455+
List<String> addModulesForBuilderVM = new ArrayList<>();
1456+
for (String module : addModules) {
1457+
Path jarPath = modules.get(module);
1458+
if (jarPath == null) {
1459+
// boot module
1460+
addModulesForBuilderVM.add(module);
1461+
}
1462+
}
1463+
1464+
if (!addModulesForBuilderVM.isEmpty()) {
1465+
arguments.add(DefaultOptionHandler.addModulesOption + "=" + String.join(",", addModulesForBuilderVM));
1466+
}
1467+
}
1468+
1469+
arguments.addAll(config.getGeneratorMainClass());
1470+
1471+
if (IS_AOT && OS.getCurrent().hasProcFS) {
1472+
/*
1473+
* GR-8254: Ensure image-building VM shuts down even if native-image dies unexpected
1474+
* (e.g. using CTRL-C in Gradle daemon mode)
1475+
*/
1476+
arguments.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID()));
1477+
}
1478+
1479+
/*
1480+
* Workaround for GR-47186: Native image cannot handle modules on the image module path,
1481+
* that are also already installed in the JDK as boot module. As a workaround we filter all
1482+
* modules from the module-path that are either already installed in the JDK as boot module,
1483+
* or were explicitly added to the builder module-path.
1484+
*
1485+
* First compute all module-jar paths that are not on the builder module-path.
1486+
*/
1487+
Set<Path> nonBuilderModulePaths = modules.values().stream()
1488+
.filter(Objects::nonNull)
1489+
.filter(Predicate.not(mp::contains))
1490+
.collect(Collectors.toSet());
1491+
1492+
/*
1493+
* Now we need to filter the substituted module path list for all the modules that may
1494+
* remain on the module-path.
1495+
*
1496+
* This should normally not be necessary, as the nonBuilderModulePaths should already be the
1497+
* set of jar files for the image module path. Nevertheless, we use the original definition
1498+
* of the module path to preserve the order of the original module path and as a precaution
1499+
* to protect against --list-modules returning too many modules.
1500+
*/
1501+
List<Path> finalImageModulePath = substitutedImageModulePath.stream()
1502+
.filter(nonBuilderModulePaths::contains)
1503+
.toList();
1504+
14541505
List<String> finalImageBuilderArgs = createImageBuilderArgs(finalImageArgs, finalImageClassPath, finalImageModulePath);
14551506

14561507
/* Construct ProcessBuilder command from final arguments */
14571508
List<String> command = new ArrayList<>();
1458-
String javaExecutable = canonicalize(config.getJavaExecutable()).toString();
14591509
command.add(javaExecutable);
14601510
command.add(createVMInvocationArgumentFile(arguments));
14611511
command.add(createImageBuilderArgumentFile(finalImageBuilderArgs));
@@ -1520,6 +1570,68 @@ protected int buildImage(List<String> javaArgs, LinkedHashSet<Path> cp, LinkedHa
15201570
}
15211571
}
15221572

1573+
/**
1574+
* Resolves and lists all modules given a module path.
1575+
*
1576+
* @see #callListModules(String, List)
1577+
*/
1578+
private static Map<String, Path> listModulesFromPath(String javaExecutable, Collection<Path> modulePath) {
1579+
if (modulePath.isEmpty()) {
1580+
return Map.of();
1581+
}
1582+
String modulePathEntries = modulePath.stream()
1583+
.map(Path::toString)
1584+
.collect(Collectors.joining(File.pathSeparator));
1585+
return callListModules(javaExecutable, List.of("-p", modulePathEntries));
1586+
}
1587+
1588+
/**
1589+
* Calls <code>java $arguments --list-modules</code> to list all modules and parse the output.
1590+
* The output consists of a map with module name as key and {@link Path} to jar file if the
1591+
* module is not installed as part of the JDK. If the module is installed as part of the
1592+
* jdk/boot-layer then a <code>null</code> path will be returned.
1593+
* <p>
1594+
* This is a much more robust solution then trying to parse the JDK file structure manually.
1595+
*/
1596+
private static Map<String, Path> callListModules(String javaExecutable, List<String> arguments) {
1597+
Process listModulesProcess = null;
1598+
Map<String, Path> result = new LinkedHashMap<>();
1599+
try {
1600+
var pb = new ProcessBuilder(javaExecutable);
1601+
pb.command().addAll(arguments);
1602+
pb.command().add("--list-modules");
1603+
pb.environment().clear();
1604+
listModulesProcess = pb.start();
1605+
try (var br = new BufferedReader(new InputStreamReader(listModulesProcess.getInputStream()))) {
1606+
while (true) {
1607+
var line = br.readLine();
1608+
if (line == null) {
1609+
break;
1610+
}
1611+
String[] splitString = StringUtil.split(line, " ", 3);
1612+
String[] splitModuleNameAndVersion = StringUtil.split(splitString[0], "@", 2);
1613+
Path externalPath = null;
1614+
if (splitString.length > 1) {
1615+
String pathURI = splitString[1]; // url: file://path/to/file
1616+
externalPath = Path.of(URI.create(pathURI)).toAbsolutePath();
1617+
}
1618+
result.put(splitModuleNameAndVersion[0], externalPath);
1619+
}
1620+
}
1621+
int exitStatus = listModulesProcess.waitFor();
1622+
if (exitStatus != 0) {
1623+
throw showError("Determining image-builder observable modules failed (Exit status %d).".formatted(exitStatus));
1624+
}
1625+
} catch (IOException | InterruptedException e) {
1626+
throw showError(e.getMessage());
1627+
} finally {
1628+
if (listModulesProcess != null) {
1629+
listModulesProcess.destroy();
1630+
}
1631+
}
1632+
return result;
1633+
}
1634+
15231635
/**
15241636
* Adds a shutdown hook to kill the image builder process if it's still alive.
15251637
*

0 commit comments

Comments
 (0)