28
28
import static com .oracle .svm .core .graal .code .SubstrateBackend .SubstrateMarkId .PROLOGUE_DECD_RSP ;
29
29
import static com .oracle .svm .core .graal .code .SubstrateBackend .SubstrateMarkId .PROLOGUE_END ;
30
30
import static com .oracle .svm .core .util .VMError .shouldNotReachHere ;
31
+ import static com .oracle .svm .core .util .VMError .unsupportedFeature ;
31
32
import static jdk .graal .compiler .core .common .GraalOptions .ZapStackOnMethodEntry ;
32
33
import static jdk .graal .compiler .lir .LIRInstruction .OperandFlag .REG ;
33
34
import static jdk .graal .compiler .lir .LIRValueUtil .asConstantValue ;
36
37
import static jdk .vm .ci .aarch64 .AArch64 .sp ;
37
38
import static jdk .vm .ci .code .ValueUtil .asRegister ;
38
39
40
+ import java .util .Arrays ;
41
+ import java .util .Collection ;
39
42
import java .util .function .BiConsumer ;
40
43
41
- import com .oracle .svm .core .interpreter .InterpreterSupport ;
42
- import com .oracle .svm .core .aarch64 .SubstrateAArch64MacroAssembler ;
43
44
import org .graalvm .nativeimage .ImageSingletons ;
44
45
45
46
import com .oracle .svm .core .FrameAccess ;
46
47
import com .oracle .svm .core .ReservedRegisters ;
47
48
import com .oracle .svm .core .SubstrateOptions ;
48
49
import com .oracle .svm .core .SubstrateUtil ;
50
+ import com .oracle .svm .core .aarch64 .SubstrateAArch64MacroAssembler ;
49
51
import com .oracle .svm .core .config .ConfigurationValues ;
50
52
import com .oracle .svm .core .deopt .Deoptimizer ;
53
+ import com .oracle .svm .core .graal .code .AssignedLocation ;
51
54
import com .oracle .svm .core .graal .code .PatchConsumerFactory ;
52
55
import com .oracle .svm .core .graal .code .SubstrateBackend ;
53
56
import com .oracle .svm .core .graal .code .SubstrateCallingConvention ;
68
71
import com .oracle .svm .core .heap .ReferenceAccess ;
69
72
import com .oracle .svm .core .heap .SubstrateReferenceMapBuilder ;
70
73
import com .oracle .svm .core .imagelayer .ImageLayerBuildingSupport ;
74
+ import com .oracle .svm .core .interpreter .InterpreterSupport ;
71
75
import com .oracle .svm .core .meta .CompressedNullConstant ;
72
76
import com .oracle .svm .core .meta .SharedField ;
73
77
import com .oracle .svm .core .meta .SharedMethod ;
@@ -243,16 +247,19 @@ public static class SubstrateAArch64IndirectCallOp extends AArch64Call.IndirectC
243
247
@ Temp ({REG }) private Value linkReg ;
244
248
private final BiConsumer <CompilationResultBuilder , Integer > offsetRecorder ;
245
249
250
+ @ Def ({REG }) private Value [] multipleResults ;
251
+
246
252
public SubstrateAArch64IndirectCallOp (ResolvedJavaMethod callTarget , Value result , Value [] parameters , Value [] temps , Value targetAddress ,
247
253
LIRFrameState state , Value javaFrameAnchor , int newThreadStatus , boolean destroysCallerSavedRegisters , Value exceptionTemp ,
248
- BiConsumer <CompilationResultBuilder , Integer > offsetRecorder ) {
254
+ BiConsumer <CompilationResultBuilder , Integer > offsetRecorder , Value [] multipleResults ) {
249
255
super (TYPE , callTarget , result , parameters , temps , targetAddress , state );
250
256
this .javaFrameAnchor = javaFrameAnchor ;
251
257
this .newThreadStatus = newThreadStatus ;
252
258
this .destroysCallerSavedRegisters = destroysCallerSavedRegisters ;
253
259
this .exceptionTemp = exceptionTemp ;
254
260
this .linkReg = lr .asValue (LIRKind .value (AArch64Kind .QWORD ));
255
261
this .offsetRecorder = offsetRecorder ;
262
+ this .multipleResults = multipleResults ;
256
263
}
257
264
258
265
@ Override
@@ -561,8 +568,9 @@ protected void emitForeignCallOp(ForeignCallLinkage linkage, Value targetAddress
561
568
if (shouldEmitOnlyIndirectCalls ()) {
562
569
RegisterValue targetRegister = AArch64 .lr .asValue (FrameAccess .getWordStamp ().getLIRKind (getLIRKindTool ()));
563
570
emitMove (targetRegister , targetAddress );
571
+ Value [] multipleResults = new Value [0 ];
564
572
append (new SubstrateAArch64IndirectCallOp (targetMethod , result , arguments , temps , targetRegister , info , Value .ILLEGAL , StatusSupport .STATUS_ILLEGAL ,
565
- getDestroysCallerSavedRegisters (targetMethod ), exceptionTemp , null ));
573
+ getDestroysCallerSavedRegisters (targetMethod ), exceptionTemp , null , multipleResults ));
566
574
} else {
567
575
assert targetAddress == null ;
568
576
append (new SubstrateAArch64DirectCallOp (targetMethod , result , arguments , temps , info , Value .ILLEGAL , StatusSupport .STATUS_ILLEGAL ,
@@ -714,6 +722,25 @@ protected DebugInfoBuilder createDebugInfoBuilder(StructuredGraph graph, NodeVal
714
722
return new SubstrateDebugInfoBuilder (graph , gen .getProviders ().getMetaAccessExtensionProvider (), nodeValueMap );
715
723
}
716
724
725
+ @ Override
726
+ public Value [] visitInvokeArguments (CallingConvention invokeCc , Collection <ValueNode > arguments ) {
727
+ Value [] values = super .visitInvokeArguments (invokeCc , arguments );
728
+ SubstrateCallingConventionType type = (SubstrateCallingConventionType ) ((SubstrateCallingConvention ) invokeCc ).getType ();
729
+
730
+ if (type .usesReturnBuffer ()) {
731
+ /*
732
+ * We save the return buffer so that it can be accessed after the call.
733
+ */
734
+ assert values .length > 0 ;
735
+ Value returnBuffer = values [0 ];
736
+ Variable saved = gen .newVariable (returnBuffer .getValueKind ());
737
+ gen .append (gen .getSpillMoveFactory ().createMove (saved , returnBuffer ));
738
+ values [0 ] = saved ;
739
+ }
740
+
741
+ return values ;
742
+ }
743
+
717
744
private boolean getDestroysCallerSavedRegisters (ResolvedJavaMethod targetMethod ) {
718
745
return ((SubstrateAArch64LIRGenerator ) gen ).getDestroysCallerSavedRegisters (targetMethod );
719
746
}
@@ -761,14 +788,44 @@ public BiConsumer<CompilationResultBuilder, Integer> getOffsetRecorder(@Suppress
761
788
return null ;
762
789
}
763
790
791
+ private static AllocatableValue asReturnedValue (AssignedLocation assignedLocation ) {
792
+ assert assignedLocation .assignsToRegister ();
793
+ Register .RegisterCategory category = assignedLocation .register ().getRegisterCategory ();
794
+ LIRKind kind ;
795
+ if (category .equals (AArch64 .CPU )) {
796
+ kind = LIRKind .value (AArch64Kind .QWORD );
797
+ } else if (category .equals (AArch64 .SIMD )) {
798
+ kind = LIRKind .value (AArch64Kind .V128_QWORD );
799
+ } else {
800
+ throw unsupportedFeature ("Register category " + category + " should not be used for returns spanning multiple registers." );
801
+ }
802
+ return assignedLocation .register ().asValue (kind );
803
+ }
804
+
764
805
@ Override
765
806
protected void emitInvoke (LoweredCallTargetNode callTarget , Value [] parameters , LIRFrameState callState , Value result ) {
807
+ var cc = (SubstrateCallingConventionType ) callTarget .callType ();
766
808
verifyCallTarget (callTarget );
767
809
if (callTarget instanceof ComputedIndirectCallTargetNode ) {
810
+ assert !cc .customABI ();
768
811
emitComputedIndirectCall ((ComputedIndirectCallTargetNode ) callTarget , result , parameters , AllocatableValue .NONE , callState );
769
812
} else {
770
813
super .emitInvoke (callTarget , parameters , callState , result );
771
814
}
815
+ if (cc .usesReturnBuffer ()) {
816
+ /*
817
+ * The buffer argument was saved in visitInvokeArguments, so that the value was not
818
+ * killed by the call.
819
+ */
820
+ Value returnBuffer = parameters [0 ];
821
+ long offset = 0 ;
822
+ for (AssignedLocation ret : cc .returnSaving ) {
823
+ Value saveLocation = gen .getArithmetic ().emitAdd (returnBuffer , gen .emitJavaConstant (JavaConstant .forLong (offset )), false );
824
+ AllocatableValue returnedValue = asReturnedValue (ret );
825
+ gen .getArithmetic ().emitStore (returnedValue .getValueKind (), saveLocation , returnedValue , callState , MemoryOrderMode .PLAIN );
826
+ offset += returnedValue .getPlatformKind ().getSizeInBytes ();
827
+ }
828
+ }
772
829
}
773
830
774
831
@ Override
@@ -785,8 +842,17 @@ protected void emitIndirectCall(IndirectCallTargetNode callTarget, Value result,
785
842
AllocatableValue targetAddress = targetRegister .asValue (FrameAccess .getWordStamp ().getLIRKind (getLIRGeneratorTool ().getLIRKindTool ()));
786
843
gen .emitMove (targetAddress , operand (callTarget .computedAddress ()));
787
844
ResolvedJavaMethod targetMethod = callTarget .targetMethod ();
845
+ SubstrateCallingConventionType cc = (SubstrateCallingConventionType ) callTarget .callType ();
846
+
847
+ Value [] multipleResults = new Value [0 ];
848
+ if (cc .customABI () && cc .usesReturnBuffer ()) {
849
+ multipleResults = Arrays .stream (cc .returnSaving )
850
+ .map (SubstrateAArch64NodeLIRBuilder ::asReturnedValue )
851
+ .toList ().toArray (new Value [0 ]);
852
+ }
853
+
788
854
append (new SubstrateAArch64IndirectCallOp (targetMethod , result , parameters , temps , targetAddress , callState , setupJavaFrameAnchor (callTarget ),
789
- getNewThreadStatus (callTarget ), getDestroysCallerSavedRegisters (targetMethod ), getExceptionTemp (callTarget ), getOffsetRecorder (callTarget )));
855
+ getNewThreadStatus (callTarget ), getDestroysCallerSavedRegisters (targetMethod ), getExceptionTemp (callTarget ), getOffsetRecorder (callTarget ), multipleResults ));
790
856
}
791
857
792
858
protected void emitComputedIndirectCall (ComputedIndirectCallTargetNode callTarget , Value result , Value [] parameters , Value [] temps , LIRFrameState callState ) {
@@ -1380,7 +1446,9 @@ protected void resetForEmittingCode(CompilationResultBuilder crb) {
1380
1446
@ Override
1381
1447
public LIRGenerationResult newLIRGenerationResult (CompilationIdentifier compilationId , LIR lir , RegisterAllocationConfig registerAllocationConfig , StructuredGraph graph , Object stub ) {
1382
1448
SharedMethod method = (SharedMethod ) graph .method ();
1383
- CallingConvention callingConvention = CodeUtil .getCallingConvention (getCodeCache (), method .getCallingConventionKind ().toType (false ), method , this );
1449
+ SubstrateCallingConventionKind ccKind = method .getCallingConventionKind ();
1450
+ SubstrateCallingConventionType ccType = ccKind .isCustom () ? method .getCustomCallingConventionType () : ccKind .toType (false );
1451
+ CallingConvention callingConvention = CodeUtil .getCallingConvention (getCodeCache (), ccType , method , this );
1384
1452
LIRGenerationResult lirGenerationResult = new SubstrateLIRGenerationResult (compilationId , lir , newFrameMapBuilder (registerAllocationConfig .getRegisterConfig ()), registerAllocationConfig ,
1385
1453
callingConvention , method );
1386
1454
0 commit comments