Skip to content
This repository was archived by the owner on Oct 6, 2025. It is now read-only.

Commit 03dd634

Browse files
Christopher-ChianelliChristopher-Chianelli
authored
fix: Stack effect for Python 3.11+ with, ignore missing forward references in type hints, copy extra and function attributes (#68)
- Use object.__getattribute__ instead of getattr when reading attributes when copying objects. - Functions are restored by looking up the function with the same qualified name - Extra attributes (that don't correspond to any instance attributes) are copied to the Python object - Use copy to create instances to play nice with complex Python classes - Change PythonIterator to an interface and move its implementation to DelegatePythonIterator
1 parent a573b12 commit 03dd634

31 files changed

+427
-136
lines changed

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/PythonBytecodeToJavaBytecodeTranslator.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1141,6 +1141,9 @@ public static void writeInstructionsForOpcodes(FunctionMetadata functionMetadata
11411141
Label targetLabel = exceptionTableTargetLabelMap.computeIfAbsent(exceptionBlock.targetInstruction,
11421142
offset -> new Label());
11431143

1144+
if (exceptionBlock.blockStartInstructionInclusive > exceptionBlock.targetInstruction) {
1145+
return;
1146+
}
11441147
functionMetadata.methodVisitor.visitTryCatchBlock(startLabel, endLabel, targetLabel,
11451148
Type.getInternalName(PythonBaseException.class));
11461149
});

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/builtins/GlobalBuiltins.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
import ai.timefold.jpyinterpreter.types.PythonSlice;
5353
import ai.timefold.jpyinterpreter.types.PythonString;
5454
import ai.timefold.jpyinterpreter.types.PythonSuperObject;
55-
import ai.timefold.jpyinterpreter.types.collections.PythonIterator;
55+
import ai.timefold.jpyinterpreter.types.collections.DelegatePythonIterator;
5656
import ai.timefold.jpyinterpreter.types.collections.PythonLikeDict;
5757
import ai.timefold.jpyinterpreter.types.collections.PythonLikeList;
5858
import ai.timefold.jpyinterpreter.types.collections.PythonLikeTuple;
@@ -753,7 +753,7 @@ public static PythonLikeObject enumerate(List<PythonLikeObject> positionalArgs,
753753
final AtomicReference<PythonLikeObject> currentIndex = new AtomicReference(start);
754754
final AtomicBoolean shouldCallNext = new AtomicBoolean(true);
755755

756-
return new PythonIterator(new Iterator<PythonLikeObject>() {
756+
return new DelegatePythonIterator(new Iterator<PythonLikeObject>() {
757757
@Override
758758
public boolean hasNext() {
759759
if (shouldCallNext.get()) {
@@ -790,7 +790,7 @@ public PythonLikeObject next() {
790790
});
791791
}
792792

793-
public static PythonIterator filter(List<PythonLikeObject> positionalArgs,
793+
public static DelegatePythonIterator filter(List<PythonLikeObject> positionalArgs,
794794
Map<PythonString, PythonLikeObject> keywordArgs, PythonLikeObject instance) {
795795
PythonLikeObject function;
796796
PythonLikeObject iterable;
@@ -835,7 +835,7 @@ public static PythonIterator filter(List<PythonLikeObject> positionalArgs,
835835
predicate = (PythonLikeFunction) function;
836836
}
837837

838-
return new PythonIterator(StreamSupport.stream(
838+
return new DelegatePythonIterator(StreamSupport.stream(
839839
Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED),
840840
false)
841841
.filter(element -> PythonBoolean
@@ -1088,7 +1088,7 @@ public static PythonLikeDict locals(List<PythonLikeObject> positionalArgs,
10881088
throw new ValueError("builtin locals() is not supported when executed in Java bytecode");
10891089
}
10901090

1091-
public static PythonIterator map(List<PythonLikeObject> positionalArgs,
1091+
public static DelegatePythonIterator map(List<PythonLikeObject> positionalArgs,
10921092
Map<PythonString, PythonLikeObject> keywordArgs, PythonLikeObject instance) {
10931093
PythonLikeFunction function;
10941094
List<PythonLikeObject> iterableList = new ArrayList<>();
@@ -1138,7 +1138,7 @@ public List<PythonLikeObject> next() {
11381138
}
11391139
};
11401140

1141-
return new PythonIterator(StreamSupport.stream(
1141+
return new DelegatePythonIterator(StreamSupport.stream(
11421142
Spliterators.spliteratorUnknownSize(iteratorIterator, Spliterator.ORDERED),
11431143
false)
11441144
.map(element -> function.$call(element, Map.of(), null))
@@ -1395,7 +1395,7 @@ public PythonLikeObject next() {
13951395
}
13961396
};
13971397

1398-
return new PythonIterator(reversedIterator);
1398+
return new DelegatePythonIterator(reversedIterator);
13991399
}
14001400

14011401
throw new ValueError(sequenceType + " does not has a __reversed__ method and does not implement the Sequence protocol");
@@ -1569,7 +1569,7 @@ public static PythonLikeObject vars(List<PythonLikeObject> positionalArgs,
15691569
return positionalArgs.get(0).$getAttributeOrError("__dict__");
15701570
}
15711571

1572-
public static PythonIterator zip(List<PythonLikeObject> positionalArgs,
1572+
public static DelegatePythonIterator zip(List<PythonLikeObject> positionalArgs,
15731573
Map<PythonString, PythonLikeObject> keywordArgs, PythonLikeObject instance) {
15741574
List<PythonLikeObject> iterableList = positionalArgs;
15751575
boolean isStrict = false;
@@ -1596,7 +1596,7 @@ public static PythonIterator zip(List<PythonLikeObject> positionalArgs,
15961596

15971597
if (iteratorList.isEmpty()) {
15981598
// Return an empty iterator if there are no iterators
1599-
return new PythonIterator(iteratorList.iterator());
1599+
return new DelegatePythonIterator(iteratorList.iterator());
16001600
}
16011601

16021602
Iterator<List<PythonLikeObject>> iteratorIterator = new Iterator<List<PythonLikeObject>>() {
@@ -1633,7 +1633,7 @@ public List<PythonLikeObject> next() {
16331633
}
16341634
};
16351635

1636-
return new PythonIterator(iteratorIterator);
1636+
return new DelegatePythonIterator(iteratorIterator);
16371637
}
16381638

16391639
public static PythonLikeFunction importFunction(PythonInterpreter pythonInterpreter) {

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/implementors/ExceptionImplementor.java‎

Lines changed: 65 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ public static void startExceptOrFinally(FunctionMetadata functionMetadata, Stack
244244
}
245245
}
246246

247-
public static void startWith(int jumpTarget, FunctionMetadata functionMetadata,
247+
public static void setupWith(int jumpTarget, FunctionMetadata functionMetadata,
248248
StackMetadata stackMetadata) {
249249
MethodVisitor methodVisitor = functionMetadata.methodVisitor;
250250

@@ -301,6 +301,50 @@ public static void startWith(int jumpTarget, FunctionMetadata functionMetadata,
301301
// cannot free, since try block store stack in locals => freeing enterResult messes up indexing of locals
302302
}
303303

304+
public static void beforeWith(FunctionMetadata functionMetadata,
305+
StackMetadata stackMetadata) {
306+
MethodVisitor methodVisitor = functionMetadata.methodVisitor;
307+
308+
methodVisitor.visitInsn(Opcodes.DUP); // duplicate context_manager twice; need one for __enter__, two for __exit__
309+
methodVisitor.visitInsn(Opcodes.DUP);
310+
311+
// First load the method __exit__
312+
methodVisitor.visitMethodInsn(Opcodes.INVOKEINTERFACE, Type.getInternalName(PythonLikeObject.class),
313+
"$getType", Type.getMethodDescriptor(Type.getType(PythonLikeType.class)),
314+
true);
315+
methodVisitor.visitLdcInsn("__exit__");
316+
methodVisitor.visitMethodInsn(Opcodes.INVOKEINTERFACE, Type.getInternalName(PythonLikeObject.class),
317+
"$getAttributeOrError",
318+
Type.getMethodDescriptor(Type.getType(PythonLikeObject.class), Type.getType(String.class)),
319+
true);
320+
methodVisitor.visitTypeInsn(Opcodes.CHECKCAST, Type.getInternalName(PythonLikeFunction.class));
321+
322+
// bind it to the context_manager
323+
methodVisitor.visitTypeInsn(Opcodes.NEW, Type.getInternalName(BoundPythonLikeFunction.class));
324+
methodVisitor.visitInsn(Opcodes.DUP_X2);
325+
methodVisitor.visitInsn(Opcodes.DUP_X2);
326+
methodVisitor.visitInsn(Opcodes.POP);
327+
methodVisitor.visitMethodInsn(Opcodes.INVOKESPECIAL, Type.getInternalName(BoundPythonLikeFunction.class),
328+
"<init>", Type.getMethodDescriptor(Type.VOID_TYPE, Type.getType(PythonLikeObject.class),
329+
Type.getType(PythonLikeFunction.class)),
330+
false);
331+
332+
// Swap __exit__ method with duplicated context_manager
333+
methodVisitor.visitInsn(Opcodes.SWAP);
334+
335+
// Call __enter__
336+
DunderOperatorImplementor.unaryOperator(methodVisitor, stackMetadata, PythonUnaryOperator.ENTER);
337+
338+
int enterResult = stackMetadata.localVariableHelper.newLocal();
339+
340+
// store enter result in temp, so it does not get saved in try block
341+
stackMetadata.localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), enterResult);
342+
343+
// Push enter result back to the stack
344+
stackMetadata.localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), enterResult);
345+
// cannot free, since try block store stack in locals => freeing enterResult messes up indexing of locals
346+
}
347+
304348
public static void startExceptBlock(FunctionMetadata functionMetadata, StackMetadata stackMetadata,
305349
ExceptionBlock exceptionBlock) {
306350
// In Python 3.11 and above, the stack here is
@@ -362,12 +406,18 @@ public static void handleExceptionInWith(FunctionMetadata functionMetadata, Stac
362406
int instruction = localVariableHelper.newLocal();
363407
int exitFunction = localVariableHelper.newLocal();
364408

365-
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exception);
366-
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exceptionArgs);
367-
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), traceback);
368-
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), label);
369-
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), stackSize);
370-
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), instruction);
409+
if (functionMetadata.pythonCompiledFunction.pythonVersion.isBefore(PythonVersion.PYTHON_3_11)) {
410+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exception);
411+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exceptionArgs);
412+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), traceback);
413+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), label);
414+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), stackSize);
415+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), instruction);
416+
} else {
417+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exception);
418+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exceptionArgs);
419+
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), traceback);
420+
}
371421
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exitFunction);
372422

373423
// load exitFunction
@@ -424,14 +474,16 @@ public static void handleExceptionInWith(FunctionMetadata functionMetadata, Stac
424474
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), exitFunction);
425475
methodVisitor.visitInsn(Opcodes.SWAP);
426476

427-
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), instruction);
428-
methodVisitor.visitInsn(Opcodes.SWAP);
477+
if (functionMetadata.pythonCompiledFunction.pythonVersion.isBefore(PythonVersion.PYTHON_3_11)) {
478+
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), instruction);
479+
methodVisitor.visitInsn(Opcodes.SWAP);
429480

430-
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), stackSize);
431-
methodVisitor.visitInsn(Opcodes.SWAP);
481+
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), stackSize);
482+
methodVisitor.visitInsn(Opcodes.SWAP);
432483

433-
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), label);
434-
methodVisitor.visitInsn(Opcodes.SWAP);
484+
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), label);
485+
methodVisitor.visitInsn(Opcodes.SWAP);
486+
}
435487

436488
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), traceback);
437489
methodVisitor.visitInsn(Opcodes.SWAP);

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/implementors/JavaPythonTypeConversionImplementor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import ai.timefold.jpyinterpreter.types.PythonLikeType;
2323
import ai.timefold.jpyinterpreter.types.PythonNone;
2424
import ai.timefold.jpyinterpreter.types.PythonString;
25+
import ai.timefold.jpyinterpreter.types.collections.DelegatePythonIterator;
2526
import ai.timefold.jpyinterpreter.types.collections.PythonIterator;
2627
import ai.timefold.jpyinterpreter.types.collections.PythonLikeDict;
2728
import ai.timefold.jpyinterpreter.types.collections.PythonLikeFrozenSet;
@@ -94,7 +95,7 @@ public static PythonLikeObject wrapJavaObject(Object object, Map<Object, PythonL
9495
}
9596

9697
if (object instanceof Iterator) {
97-
return new PythonIterator((Iterator) object);
98+
return new DelegatePythonIterator<>((Iterator) object);
9899
}
99100

100101
if (object instanceof List) {

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/implementors/StackManipulationImplementor.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import ai.timefold.jpyinterpreter.FunctionMetadata;
88
import ai.timefold.jpyinterpreter.LocalVariableHelper;
99
import ai.timefold.jpyinterpreter.PythonLikeObject;
10+
import ai.timefold.jpyinterpreter.PythonVersion;
1011
import ai.timefold.jpyinterpreter.StackMetadata;
1112
import ai.timefold.jpyinterpreter.ValueSourceInfo;
1213

@@ -296,12 +297,22 @@ public static void restoreExceptionTableStack(FunctionMetadata functionMetadata,
296297
LocalVariableHelper localVariableHelper = stackMetadata.localVariableHelper;
297298

298299
localVariableHelper.readExceptionTableTargetStack(methodVisitor, exceptionBlock.getTargetInstruction());
300+
299301
for (int i = 0; i < exceptionBlock.getStackDepth(); i++) {
300302
methodVisitor.visitInsn(Opcodes.DUP);
301303
methodVisitor.visitLdcInsn(i);
302304
methodVisitor.visitInsn(Opcodes.AALOAD);
303-
methodVisitor.visitTypeInsn(Opcodes.CHECKCAST,
304-
stackMetadata.getTypeAtStackIndex(exceptionBlock.getStackDepth() - i - 1).getJavaTypeInternalName());
305+
306+
if (functionMetadata.pythonCompiledFunction.pythonVersion.isBefore(PythonVersion.PYTHON_3_11) ||
307+
functionMetadata.pythonCompiledFunction.pythonVersion.isAtLeast(PythonVersion.PYTHON_3_12)) {
308+
// Not a 3.11 python version
309+
methodVisitor.visitTypeInsn(Opcodes.CHECKCAST,
310+
stackMetadata.getTypeAtStackIndex(exceptionBlock.getStackDepth() - i - 1).getJavaTypeInternalName());
311+
} else {
312+
// A 3.11 python version
313+
methodVisitor.visitTypeInsn(Opcodes.CHECKCAST,
314+
Type.getInternalName(PythonLikeObject.class));
315+
}
305316
methodVisitor.visitInsn(Opcodes.SWAP);
306317
}
307318
methodVisitor.visitInsn(Opcodes.POP);

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/opcodes/descriptor/ExceptionOpDescriptor.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import ai.timefold.jpyinterpreter.PythonBytecodeInstruction;
88
import ai.timefold.jpyinterpreter.PythonVersion;
99
import ai.timefold.jpyinterpreter.opcodes.Opcode;
10+
import ai.timefold.jpyinterpreter.opcodes.exceptions.BeforeWithOpcode;
1011
import ai.timefold.jpyinterpreter.opcodes.exceptions.CheckExcMatchOpcode;
1112
import ai.timefold.jpyinterpreter.opcodes.exceptions.CleanupThrowOpcode;
1213
import ai.timefold.jpyinterpreter.opcodes.exceptions.LoadAssertionErrorOpcode;
@@ -64,6 +65,7 @@ public enum ExceptionOpDescriptor implements OpcodeDescriptor {
6465
WITH_EXCEPT_START(WithExceptStartOpcode::new),
6566
SETUP_FINALLY(SetupFinallyOpcode::new, JumpUtils::getRelativeTarget),
6667

68+
BEFORE_WITH(BeforeWithOpcode::new),
6769
SETUP_WITH(SetupWithOpcode::new, JumpUtils::getRelativeTarget),
6870
CLEANUP_THROW(CleanupThrowOpcode::new);
6971

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/opcodes/exceptions/BeforeWithOpcode.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package ai.timefold.jpyinterpreter.opcodes.exceptions;
2+
3+
import ai.timefold.jpyinterpreter.FunctionMetadata;
4+
import ai.timefold.jpyinterpreter.PythonBytecodeInstruction;
5+
import ai.timefold.jpyinterpreter.StackMetadata;
6+
import ai.timefold.jpyinterpreter.ValueSourceInfo;
7+
import ai.timefold.jpyinterpreter.implementors.ExceptionImplementor;
8+
import ai.timefold.jpyinterpreter.opcodes.AbstractOpcode;
9+
import ai.timefold.jpyinterpreter.types.BuiltinTypes;
10+
import ai.timefold.jpyinterpreter.types.PythonLikeFunction;
11+
12+
public class BeforeWithOpcode extends AbstractOpcode {
13+
14+
public BeforeWithOpcode(PythonBytecodeInstruction instruction) {
15+
super(instruction);
16+
}
17+
18+
@Override
19+
protected StackMetadata getStackMetadataAfterInstruction(FunctionMetadata functionMetadata, StackMetadata stackMetadata) {
20+
return stackMetadata
21+
.pop()
22+
.push(ValueSourceInfo.of(this, PythonLikeFunction.getFunctionType(), stackMetadata.getTOSValueSource()))
23+
.push(ValueSourceInfo.of(this, BuiltinTypes.BASE_TYPE, stackMetadata.getTOSValueSource()));
24+
}
25+
26+
@Override
27+
public void implement(FunctionMetadata functionMetadata, StackMetadata stackMetadata) {
28+
ExceptionImplementor.beforeWith(functionMetadata, stackMetadata);
29+
}
30+
}

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/opcodes/exceptions/SetupWithOpcode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,6 @@ public List<StackMetadata> getStackMetadataAfterInstructionForBranches(FunctionM
4949

5050
@Override
5151
public void implement(FunctionMetadata functionMetadata, StackMetadata stackMetadata) {
52-
ExceptionImplementor.startWith(jumpTarget, functionMetadata, stackMetadata);
52+
ExceptionImplementor.setupWith(jumpTarget, functionMetadata, stackMetadata);
5353
}
5454
}

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/opcodes/exceptions/WithExceptStartOpcode.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import ai.timefold.jpyinterpreter.FunctionMetadata;
44
import ai.timefold.jpyinterpreter.PythonBytecodeInstruction;
5+
import ai.timefold.jpyinterpreter.PythonVersion;
56
import ai.timefold.jpyinterpreter.StackMetadata;
67
import ai.timefold.jpyinterpreter.ValueSourceInfo;
78
import ai.timefold.jpyinterpreter.implementors.ExceptionImplementor;
@@ -16,7 +17,10 @@ public WithExceptStartOpcode(PythonBytecodeInstruction instruction) {
1617

1718
@Override
1819
protected StackMetadata getStackMetadataAfterInstruction(FunctionMetadata functionMetadata, StackMetadata stackMetadata) {
19-
// TODO: this might need updating to handle Python 3.11
20+
if (functionMetadata.pythonCompiledFunction.pythonVersion.isAtLeast(PythonVersion.PYTHON_3_11)) {
21+
return stackMetadata
22+
.push(ValueSourceInfo.of(this, BuiltinTypes.BASE_TYPE, stackMetadata.getValueSourceForStackIndex(1)));
23+
}
2024
return stackMetadata
2125
.push(ValueSourceInfo.of(this, BuiltinTypes.BASE_TYPE, stackMetadata.getValueSourceForStackIndex(6)));
2226
}

‎jpyinterpreter/src/main/java/ai/timefold/jpyinterpreter/types/AbstractPythonLikeObject.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@ public void setAttribute(String attributeName, PythonLikeObject value) {
5050
__dir__.put(attributeName, value);
5151
}
5252

53+
public Map<String, PythonLikeObject> getExtraAttributeMap() {
54+
return __dir__;
55+
}
56+
5357
@Override
5458
public String toString() {
5559
return $method$__str__().toString();

0 commit comments

Comments
 (0)