From 4e2b6ab399ebe1de942810d4d0b26b8cda59f152 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 15 Apr 2026 15:26:15 +0200 Subject: [PATCH 01/26] Grammar addition: Token and NumericLiteral for imaginary numbers in python. --- src/main/grammars/de/monticore/Python.mc4 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 0f4e87d..5f64e96 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -74,6 +74,8 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; PyFloatLiteral implements NumericLiteral <200> = PyFloat; + token PyImaginaryNumber = (DigitsPart | PyFloat)? ('+' | '-') DigitsPart "j"; + PyImaginaryNumberLiteral implements NumericLiteral <95> = PyImaginaryNumber; // PEP 515 token DigitsPart = Digit ('_'? Digit)*; From e16c5de4091c18fdaabd4777d4c0f50c9df83063 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 15 Apr 2026 15:29:24 +0200 Subject: [PATCH 02/26] Grammar addition: nonlocal statements yield from statements --- src/main/grammars/de/monticore/Python.mc4 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 5f64e96..158b43d 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -76,6 +76,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit token PyImaginaryNumber = (DigitsPart | PyFloat)? ('+' | '-') DigitsPart "j"; PyImaginaryNumberLiteral implements NumericLiteral <95> = PyImaginaryNumber; + // PEP 515 token DigitsPart = Digit ('_'? Digit)*; @@ -206,7 +207,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ); GlobalVariableDeclaration implements Statement = "global" Name (":" TypeAnnotation)? STATEMENT_END; - + NonLocalVariableDeclaration implements Statement = "nonlocal" (Name || ",")+ STATEMENT_END; MultiVariableDeclaration implements Statement = (Name || ",")+ ","? "=" Expression STATEMENT_END; ParenMultiVariableDeclaration implements Statement = "(" (Name || ",")+ ","? ")" "=" Expression STATEMENT_END; @@ -242,6 +243,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ReturnStatement implements Statement = "return" (Expression || ",")* ","? STATEMENT_END; YieldStatement implements Statement = "yield" (Expression || ",")* ","? STATEMENT_END; + YieldFromStatement implements Statement = "yield" "from" Expression; RaiseStatement implements Statement = "raise" (Expression ("from" Name)?)? STATEMENT_END; ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; From 0f87c1f25a675c7f15381d1a7bb207f555033f75 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 15 Apr 2026 15:31:03 +0200 Subject: [PATCH 03/26] Grammar extension: Add support for **=,@=,//= assignmentexpressions. --- src/main/grammars/de/monticore/Python.mc4 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 158b43d..f18a377 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -304,6 +304,13 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit | "^=" | ">>=" | ">>>=" | "<<=" | "%=" ] right:Expression; + @Override + AssignmentExpression implements Expression <60> = + left:Expression + operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" + | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] + right:Expression; + UnpackingAssignmentExpression implements Expression <60> = "(" left:Expression ("," left:Expression)* ","? ")" "=" right: Expression; From 15109dda9b30209459b2b0014a2e26a5a1c84869 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 15 Apr 2026 15:58:02 +0200 Subject: [PATCH 04/26] Grammar changes: Global declarations Removing support for AnnotedAssignments apart from "=" --- src/main/grammars/de/monticore/Python.mc4 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index f18a377..9318f58 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -206,7 +206,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit (":" TypeAnnotation) ); - GlobalVariableDeclaration implements Statement = "global" Name (":" TypeAnnotation)? STATEMENT_END; + GlobalVariableDeclaration implements Statement = "global" (Name || ",")+ STATEMENT_END; NonLocalVariableDeclaration implements Statement = "nonlocal" (Name || ",")+ STATEMENT_END; MultiVariableDeclaration implements Statement = (Name || ",")+ ","? "=" Expression STATEMENT_END; ParenMultiVariableDeclaration implements Statement = "(" (Name || ",")+ ","? ")" "=" Expression STATEMENT_END; @@ -300,8 +300,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit AnnotatedAssignmentExpression implements Expression <60> = left:Expression ":" annotated: TypeAnnotation - operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" - | "^=" | ">>=" | ">>>=" | "<<=" | "%=" ] + "=" right:Expression; @Override From 606a5f6caaeecaaf43eceea97704b1d9cc37e782 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 15 Apr 2026 16:05:14 +0200 Subject: [PATCH 05/26] Grammar changes: Avoiding Nullpointer exceptions, After parsing a ExceptStatement a NullPointerException could happen during prettyprinting. --- src/main/grammars/de/monticore/Python.mc4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 9318f58..ec1aab8 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -188,7 +188,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ) ; - ExceptStatement = "except" (PyQualifiedName? | "(" (PyQualifiedName || ",")+ ")") ("as" alias:Name)? ":" StatementBlock; + ExceptStatement = "except" (PyQualifiedName | "(" (PyQualifiedName || ",")+ ")")? ("as" alias:Name)? ":" StatementBlock; FinallyStatement = "finally" ":" finallyStatement:StatementBlock; // with open file statement @@ -243,7 +243,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ReturnStatement implements Statement = "return" (Expression || ",")* ","? STATEMENT_END; YieldStatement implements Statement = "yield" (Expression || ",")* ","? STATEMENT_END; - YieldFromStatement implements Statement = "yield" "from" Expression; + YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; RaiseStatement implements Statement = "raise" (Expression ("from" Name)?)? STATEMENT_END; ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; From 8c519eff8bcdf027ae421726fc36929f0bde2dd1 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Fri, 17 Apr 2026 17:33:46 +0200 Subject: [PATCH 06/26] Change to preprocessor: Always ignore continueLineTokens --- ...sedWhitespacePreprocessingTokenSource.java | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java b/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java index 18b9afd..4879db3 100644 --- a/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java +++ b/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java @@ -23,33 +23,35 @@ public class StateBasedWhitespacePreprocessingTokenSource { private final List closingParens; private Token lastToken; private Token lastEmittedToken; + private PreprocessingTokens preprocessingTokens; public StateBasedWhitespacePreprocessingTokenSource( - Pair source, - List indents, - List emptyLines, - Lexer delegate, - Token eolTokenProto, - Token incIndentTokenProto, - Token decIndentTokenProto, - PreprocessingTokens preprocessingTokens + Pair source, + List indents, + List emptyLines, + Lexer delegate, + Token eolTokenProto, + Token incIndentTokenProto, + Token decIndentTokenProto, + PreprocessingTokens preprocessingTokens ) { this.openingParens = List.of(preprocessingTokens.lparenTokenType, - preprocessingTokens.lCurlyParenTokenType, - preprocessingTokens.lSquareParenTokenType); + preprocessingTokens.lCurlyParenTokenType, + preprocessingTokens.lSquareParenTokenType); this.closingParens = List.of(preprocessingTokens.rparenTokenType, - preprocessingTokens.rCurlyParenTokenType, - preprocessingTokens.rSquareParenTokenType); + preprocessingTokens.rCurlyParenTokenType, + preprocessingTokens.rSquareParenTokenType); this.sensitiveProcessor = new WhitespaceSensitiveProcessor( - source, indents, emptyLines, delegate.getTokenFactory(), - eolTokenProto, incIndentTokenProto, decIndentTokenProto, - preprocessingTokens + source, indents, emptyLines, delegate.getTokenFactory(), + eolTokenProto, incIndentTokenProto, decIndentTokenProto, + preprocessingTokens ); this.lastToken = null; this.lastEmittedToken = null; + this.preprocessingTokens = preprocessingTokens; } public List process(Token token) { @@ -70,8 +72,12 @@ public List process(Token token) { // whitespace sensitive res = sensitiveProcessor.process(token, lastToken, lastEmittedToken); } else { - // whitespace insensitive - return List.of(token); + // always ignore the continueLineToken + if(token.getType() == preprocessingTokens.continueLineTokenType){ + return List.of(); + } else { + return List.of(token); + } } // bookkeeping of emitted tokens From 3f4fd394f70155fa33188c27f422430ddd2f030b Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Fri, 17 Apr 2026 21:14:49 +0200 Subject: [PATCH 07/26] Change to grammar: Improving ExceptStatements and allowing the passing of classes --- src/main/grammars/de/monticore/Python.mc4 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index ec1aab8..cd3c8c9 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -188,7 +188,8 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ) ; - ExceptStatement = "except" (PyQualifiedName | "(" (PyQualifiedName || ",")+ ")")? ("as" alias:Name)? ":" StatementBlock; + ExceptStatement = "except" (PyQualifiedName | "(" ExceptListing ")" | ExceptListing )? ("as" alias:Name)? ":" StatementBlock; + ExceptListing = (PyQualifiedName || ",")+ | (PyQualifiedName || "or")+; FinallyStatement = "finally" ":" finallyStatement:StatementBlock; // with open file statement @@ -344,7 +345,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit interface ClassStatement; - ClassStatementBlock = BLOCK_START ClassStatementBlockBody BLOCK_END; + ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END) | ClassStatement ; ClassStatementBlockBody = ClassStatement+; ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; From bab389d6300eaaad2bc42243de1601c72b086ca0 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Thu, 23 Apr 2026 15:23:53 +0200 Subject: [PATCH 08/26] Change to grammar: Improving ExceptStatements and allowing unsigned imaginary numbers --- src/main/grammars/de/monticore/Python.mc4 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index cd3c8c9..e190e9e 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -74,7 +74,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; PyFloatLiteral implements NumericLiteral <200> = PyFloat; - token PyImaginaryNumber = (DigitsPart | PyFloat)? ('+' | '-') DigitsPart "j"; + token PyImaginaryNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; PyImaginaryNumberLiteral implements NumericLiteral <95> = PyImaginaryNumber; // PEP 515 @@ -188,8 +188,8 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ) ; - ExceptStatement = "except" (PyQualifiedName | "(" ExceptListing ")" | ExceptListing )? ("as" alias:Name)? ":" StatementBlock; - ExceptListing = (PyQualifiedName || ",")+ | (PyQualifiedName || "or")+; + ExceptStatement = "except" (PyQualifiedName | "(" Alternatives ")" | (PyQualifiedName || "or")+ )? ("as" alias:Name)? ":" StatementBlock; + Alternatives = (PyQualifiedName || ",")+ | (PyQualifiedName || "or")+ | (PyQualifiedName || "|")+ ; FinallyStatement = "finally" ":" finallyStatement:StatementBlock; // with open file statement From e30024ce414fa0fff48696a71912995f9f1cd4b1 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Thu, 23 Apr 2026 15:31:25 +0200 Subject: [PATCH 09/26] Addition to grammar: Generics and Typehints --- src/main/grammars/de/monticore/Python.mc4 | 26 +++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index e190e9e..42cdf63 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -219,15 +219,15 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit // function declaration statement interface FunctionDeclaration extends Function = Name ; - SimpleFunctionDeclaration implements FunctionDeclaration, Statement = PyDecorator* async:"async"? "def" Name "(" FunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" + SimpleFunctionDeclaration implements FunctionDeclaration, Statement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" FunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock ; FunctionParameters = (FunctionParameter || ",")* ","?; interface FunctionParameter; - SimpleFunctionParameter implements FunctionParameter, Variable = Name (":" TypeAnnotation)?; - OptionalFunctionParameter implements FunctionParameter, Variable = Name (":" TypeAnnotation)? "=" Expression ; - VarArgFunctionParameter implements FunctionParameter, Variable = "*" Name (":" TypeAnnotation)?; - KWArgFunctionParameter implements FunctionParameter, Variable = "**" Name (":" TypeAnnotation)?; + SimpleFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)?; + OptionalFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)? "=" Expression ; + VarArgFunctionParameter implements FunctionParameter, Variable = "*" Name GenericsAnnotation? (":" TypeAnnotation)?; + KWArgFunctionParameter implements FunctionParameter, Variable = "**" Name GenericsAnnotation? (":" TypeAnnotation)?; StarFunctionParameter implements FunctionParameter = "*"; @Override @@ -247,6 +247,10 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; RaiseStatement implements Statement = "raise" (Expression ("from" Name)?)? STATEMENT_END; + // Type related statemets + TypeAnnotationStatement implements Statement = Expression ":" TypeAnnotation STATEMENT_END; + TypeAliasStatement implements Statement = key("type") Name "=" Expression STATEMENT_END; + ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; EmptyStatement implements Statement, ClassStatement = STATEMENT_END; @@ -341,7 +345,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit interface scope symbol PythonClass = Name ; // class declaration statement - ClassDeclaration implements PythonClass, Statement, ClassStatement = PyDecorator* "class" Name ( "(" ((superClasses:PyQualifiedName TypeAnnotation? | arguments:NamedArgument) ","?)* ")" )? ":" ClassStatementBlock ; + ClassDeclaration implements PythonClass, Statement, ClassStatement = PyDecorator* "class" Name GenericsAnnotation? ( "(" ((superClasses:PyQualifiedName TypeAnnotation? | arguments:NamedArgument) ","?)* ")" )? ":" ClassStatementBlock ; interface ClassStatement; @@ -350,7 +354,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; - ClassFunctionParameters = (FunctionParameter || ",")* ","?; + ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; ClassAttributes implements ClassStatement = VariableDeclaration STATEMENT_END; @@ -359,6 +363,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit /*====================================== Type Annotations ======================================*/ interface TypeAnnotation; + GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation; StringTypeAnnotation implements TypeAnnotation = StringLiteralPython; TupleTypeAnnotation implements TypeAnnotation = "(" (TypeAnnotation || ",")+ ","? ")"; QualifiedTypeAnnotation implements TypeAnnotation = type:PyQualifiedName ("[" typeParams:(TypeAnnotation || ",")* "]")?; @@ -367,4 +372,11 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ListTypeAnnotation implements TypeAnnotation = "[" TypeAnnotation "]"; EllipsisTypeAnnotation implements TypeAnnotation = "..."; ComplexTypeAnnotation implements TypeAnnotation <200> = Expression; + + /*====================================== Generics ======================================*/ + + GenericsAnnotation = "[" Generics? "]"; + Generics = (Generic || ",")+ ; + Generic = PyQualifiedName (":" TypeAnnotation)?; + } From 957631cd0f6ce3a6e32b7662fc0269a4029c9b91 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Thu, 23 Apr 2026 15:45:35 +0200 Subject: [PATCH 10/26] Changes to grammar: Minor fixes, Additional Match/Case combinations allowed Parenthesis for Expressions added --- src/main/grammars/de/monticore/Python.mc4 | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 42cdf63..a829ffd 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -255,10 +255,10 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit EmptyStatement implements Statement, ClassStatement = STATEMENT_END; - MatchStatement implements Statement = key("match") Expression ":" MatchBlock; + MatchStatement implements Statement = key("match") (Expression || ",")+ ":" MatchBlock; scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; - CaseStatement = key("case") (Expression || "|")+ ("if" condition:Expression)? ":" StatementBlock; + CaseStatement = key("case") (Expression || ",")+ ("if" condition:Expression)? ":" StatementBlock; ConditionalExecutionStatement implements Statement = "if" condition: Expression ":" Expression; @@ -266,6 +266,11 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit /*====================================== Expressions ======================================*/ + // Aliased Expression only allowed in Case Statements (Checked with CoCo). + AliasedExpression implements Expression = Expression "as" alias:Name; + + ParenthesisExpression implements Expression = "(" Expression ")"; + SpreadListExpression implements Expression = "*" Expression; SpreadMappingExpression implements Expression = "**" Expression; splittoken "**"; @@ -306,14 +311,14 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit left:Expression ":" annotated: TypeAnnotation "=" - right:Expression; + right:Expression ","?; @Override AssignmentExpression implements Expression <60> = left:Expression operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] - right:Expression; + right:Expression ","?; UnpackingAssignmentExpression implements Expression <60> = "(" left:Expression ("," left:Expression)* ","? ")" "=" @@ -349,7 +354,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit interface ClassStatement; - ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END) | ClassStatement ; + ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement; ClassStatementBlockBody = ClassStatement+; ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; From ba9e8f18fb1c038a2cca33527a1e1bb8552778ab Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Thu, 23 Apr 2026 15:51:47 +0200 Subject: [PATCH 11/26] Changes to grammar: Minor fixes of previous mistake Added CoCo for aliased expressions --- src/main/grammars/de/monticore/Python.mc4 | 4 +-- .../python/_cocos/CaseStatementVisitor.java | 36 +++++++++++++++++++ .../_cocos/ExpressionsCorrectlyAliased.java | 19 ++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 src/main/java/de/monticore/python/_cocos/CaseStatementVisitor.java create mode 100644 src/main/java/de/monticore/python/_cocos/ExpressionsCorrectlyAliased.java diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index a829ffd..f17f6ad 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -357,10 +357,10 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement; ClassStatementBlockBody = ClassStatement+; - ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; - ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; + ClassFunctionParameters = (FunctionParameter || ",")* ","?; + ClassAttributes implements ClassStatement = VariableDeclaration STATEMENT_END; ClassCommentStatement implements ClassStatement = MultiLineStringLiteral STATEMENT_END; diff --git a/src/main/java/de/monticore/python/_cocos/CaseStatementVisitor.java b/src/main/java/de/monticore/python/_cocos/CaseStatementVisitor.java new file mode 100644 index 0000000..691911b --- /dev/null +++ b/src/main/java/de/monticore/python/_cocos/CaseStatementVisitor.java @@ -0,0 +1,36 @@ +package de.monticore.python._cocos; + +import de.monticore.python._ast.*; +import de.monticore.python._visitor.PythonVisitor2; + +public class CaseStatementVisitor implements PythonVisitor2 { + private boolean inCaseStatement = false; + private int numberOfViolations = 0; + + @Override + public void visit(de.monticore.python._ast.ASTCaseStatement node){ + this.inCaseStatement = true; + } + + @Override + public void endVisit(de.monticore.python._ast.ASTCaseStatement node){ + this.inCaseStatement = false; + } + + @Override + public void visit(de.monticore.python._ast.ASTAliasedExpression node){ + if(!inCaseStatement){ + numberOfViolations++; + } + } + public int getNumberOfViolations (){ + return numberOfViolations; + } + public boolean violationsFound(){ + if(numberOfViolations>0) { + return true; + } else { + return false; + } + } +} \ No newline at end of file diff --git a/src/main/java/de/monticore/python/_cocos/ExpressionsCorrectlyAliased.java b/src/main/java/de/monticore/python/_cocos/ExpressionsCorrectlyAliased.java new file mode 100644 index 0000000..013fd76 --- /dev/null +++ b/src/main/java/de/monticore/python/_cocos/ExpressionsCorrectlyAliased.java @@ -0,0 +1,19 @@ +package de.monticore.python._cocos; +import de.monticore.python._ast.ASTPythonScript; +import de.monticore.python.PythonMill; +import de.se_rwth.commons.logging.Log; + +public class ExpressionsCorrectlyAliased implements PythonASTPythonScriptCoCo { + CaseStatementVisitor visitor = new CaseStatementVisitor(); + + @Override + public void check(de.monticore.python._ast.ASTPythonScript node){ + CaseStatementVisitor visitor = new CaseStatementVisitor(); + de.monticore.python._visitor.PythonTraverser traverser = PythonMill.traverser(); + traverser.add4Python(visitor); + node.accept(traverser); + if(visitor.violationsFound()){ + Log.error("In "+visitor.getNumberOfViolations()+" cases an alias was used in an unintended way, in example: var1 = var2 as var 3 "); + } + } +} \ No newline at end of file From 2edd789d9fb18925c2f40ef7e11d7d4069230d73 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Fri, 1 May 2026 14:00:24 +0200 Subject: [PATCH 12/26] Addition of separate fstring tokens to handle various syntax in the curly brackets. Overriding of the BooleanLiteral to allow "true" and "false" as names. Allowing various Statements in classes. Except statements now span a scope to capture the names of aliased qualified names. Aliased expression now contain a symbol to be identified. Casestatements now span a scope to capture the symbol of aliased expressions. Formatting and comments for the Python.mc4. --- .../grammars/de/monticore/MultilineString.mc4 | 3 +- src/main/grammars/de/monticore/Python.mc4 | 648 +++++++++--------- 2 files changed, 321 insertions(+), 330 deletions(-) diff --git a/src/main/grammars/de/monticore/MultilineString.mc4 b/src/main/grammars/de/monticore/MultilineString.mc4 index 04b5981..b287110 100644 --- a/src/main/grammars/de/monticore/MultilineString.mc4 +++ b/src/main/grammars/de/monticore/MultilineString.mc4 @@ -5,5 +5,6 @@ package de.monticore; // This can be fixed by adding another token, which must be defined before the String token. // Thus, this grammar must be used before MCCommonLiterals, which most langauges use. component grammar MultilineString { - token DoubleQuoteMultilineStringDelimiter = '"' '"' '"'; + token DoubleQuoteMultilineFStringDelimiter = ('f'|'F') '"' '"' '"'; + token DoubleQuoteMultilineStringDelimiter = '"' '"' '"'; } \ No newline at end of file diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index f17f6ad..a458079 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -24,364 +24,354 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit PythonScript = Statement*; /*====================================== Tokens ======================================*/ - @Override - token WS = (' ' | '\t' | '\r' | '\n' ) : -> channel(HIDDEN); - @Override - token SL_COMMENT = "#" (~('\n' | '\r' ))* : -> channel(HIDDEN); - token ByteOrderMark = '\uFEFF' : -> skip; - - /** - * The following utf-8 symbols are used to parse the Python files without having to add whitespace to the Grammar. - * In a preprocessing step code blocks are denoted with \u204f = ⦃ and \u2984 = ⦄, and lines are ended with \u204f = ⁏ - */ - token BLOCK_START = '\u2983'; - token BLOCK_END = '\u2984'; - token STATEMENT_END = ';'? '\u204f' | ';' '\u204f'?; - - // Will be filtered out by the WhitespacePreprocessingTokenSource - // Used break a line without finishing the statement - token CONTINUE_LINE_TOKEN = '\\' '\r'? '\n'; - - // === string tokens for python === - // Often (mis)used as a multiline comment but can also be used as a string literal, thus we can not skip it - token MultiLineStringToken = ((("\'\'\'") .*? ("\'\'\'")) | ((DoubleQuoteMultilineStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); - - @Override - token String = '"' (StringDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; - - token StringPython - = '\'' (StringSQCharactersPython)? '\'' - : {setText(getText().substring(1, getText().length() - 1));}; - - fragment token StringSQCharactersPython - = (StringSQCharacterPython)+; - fragment token StringDQCharactersPython - = (StringDQCharacterPython)+; - - fragment token StringSQCharacterPython - = ~ ('\''| '\\' | '\n') | PythonEscapeSequence; - fragment token StringDQCharacterPython - = ~ ('"'| '\\' | '\n') | PythonEscapeSequence; - - fragment token PythonEscapeSequence - = '\\' .; - - // === number tokens for python === - - token FloatWithExponent = (DigitsPart | PyFloat) ('e'|'E') ('+' | '-')? DigitsPart; - FloatWithExponentLiteral implements NumericLiteral <100> = FloatWithExponent; - - token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; - PyFloatLiteral implements NumericLiteral <200> = PyFloat; - - token PyImaginaryNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; - PyImaginaryNumberLiteral implements NumericLiteral <95> = PyImaginaryNumber; - - // PEP 515 - token DigitsPart = Digit ('_'? Digit)*; - - token HexNumberToken = '0' 'x' ('0'..'9' | 'a'..'f' | 'A'..'F')+; - // PEP 515: Underscores in Numeric Literals - @Override - token Digits - = Digit ('_'? Digit)*; // technically the first digit must be nonzero except for 0(_0)* + @Override + token WS = (' ' | '\t' | '\r' | '\n' ) : -> channel(HIDDEN); + @Override + token SL_COMMENT = "#" (~('\n' | '\r' ))* : -> channel(HIDDEN); + token ByteOrderMark = '\uFEFF' : -> skip; + + /** + * The following utf-8 symbols are used to parse the Python files without having to add whitespace to the Grammar. + * In a preprocessing step code blocks are denoted with \u204f = ⦃ and \u2984 = ⦄, and lines are ended with \u204f = ⁏ + */ + token BLOCK_START = '\u2983'; + token BLOCK_END = '\u2984'; + token STATEMENT_END = ';'? '\u204f' | ';' '\u204f'?; + + // Will be filtered out by the WhitespacePreprocessingTokenSource + // Used break a line without finishing the statement + token CONTINUE_LINE_TOKEN = '\\' '\r'? '\n'; + + //String tokens for python + + //Often (mis)used as a multiline comment but can also be used as a string literal, thus we can not skip it + token MultiLineStringToken = ((("\'\'\'") .*? ("\'\'\'")) | ((DoubleQuoteMultilineStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); + token MultiLineFStringToken = ((("\'\'\'") .*? ("\'\'\'")) |((DoubleQuoteMultilineFStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); + + //Double quoted Strings "Text" + @Override + token String = '"' (StringDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; + fragment token StringDQCharactersPython = (StringDQCharacterPython)+; + fragment token StringDQCharacterPython = ~ ('"' | '\\' | '\n') | PythonEscapeSequence; + + //Single quoted Strings 'Text' + token StringPython = '\'' (StringSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; + fragment token StringSQCharactersPython = (StringSQCharacterPython)+; + fragment token StringSQCharacterPython = ~ ('\''| '\\' | '\n') | PythonEscapeSequence; + + //Escape in strings "\n" + fragment token PythonEscapeSequence = '\\' .; + + //Double and single quoted strings with an f modifier f'text1{exp}text2', separate definition to allow more expressions. + token FSQStringPython + = ('f'|'F') '\'' (StringFSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; + token FDQStringPython + = ('f'|'F') '"' (StringFDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; + fragment token StringFSQCharactersPython = (StringFSQCharacterPython)+; + fragment token StringFDQCharactersPython = (StringFDQCharacterPython)+; + fragment token StringFSQCharacterPython = ~ ('\''| '\\'| '{' )| PythonEscapeSequence |InnerFString; + fragment token StringFDQCharacterPython = ~ ('"'| '\\'| '{') | PythonEscapeSequence |InnerFString; + fragment token InnerFString = '{' ~('}')* '}'; + + //number tokens for python + + token FloatWithExponent = (DigitsPart | PyFloat) ('e'|'E') ('+' | '-')? DigitsPart; + FloatWithExponentLiteral implements NumericLiteral <100> = FloatWithExponent; + + token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; + PyFloatLiteral implements NumericLiteral <200> = PyFloat; + + token PyImaginaryNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; + PyImaginaryNumberLiteral implements NumericLiteral <95> = PyImaginaryNumber; + + //PEP 515 + token DigitsPart = Digit ('_'? Digit)*; + + token HexNumberToken = '0' 'x' ('0'..'9' | 'a'..'f' | 'A'..'F')+; + + //PEP 515: Underscores in Numeric Literals + @Override + token Digits = Digit ('_'? Digit)*; // technically the first digit must be nonzero except for 0(_0)* /*====================================== Literals ======================================*/ - HexNumberLiteral implements NumericLiteral <100> = HexNumberToken; + HexNumberLiteral implements NumericLiteral <100> = HexNumberToken; - ArrayLiteral implements Literal = "[" (VariableInit || ",")* ","? "]" ; - TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ; - DictLiteral implements Literal = "{" (DictEntry || ",")* ","? "}" ; - SetLiteral implements Literal = "{" (Expression || ",")* ","? "}" ; + //Literals for Datastructures - DictEntry = key:VariableInit ":" value:VariableInit | SpreadMappingExpression; + ArrayLiteral implements Literal = "[" (VariableInit || ",")* ","? "]" ; + TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ; + DictLiteral implements Literal = "{" (DictEntry || ",")* ","? "}" ; + SetLiteral implements Literal = "{" (Expression || ",")* ","? "}" ; + DictEntry = key:VariableInit ":" value:VariableInit | SpreadMappingExpression; - StringLiteralPython implements Literal, SignedLiteral = - ( - StringModifier? - (source:StringPython | source:String) - ); + //Literals and helper-definitions regarding Strings - StringsLiteralPython implements Literal <200> = (StringLiteralPython | StringLiteral | MultiLineStringLiteral)+; + MultiLineStringLiteral implements Literal = (StringModifier)? MultiLineStringToken; + MultiLineFStringLiteral implements Literal = (StringModifier)? MultiLineFStringToken; + StringLiteralPython implements Literal, SignedLiteral = + ( + StringModifier? + (source:StringPython | source:String |source:Char) + ) + | fsource: FStringPython; - StringModifier = /*{cmpTokenRegEx(1, "(f|r|b|u|F|R|B|U)+")}?*/ type:Name; - MultiLineStringLiteral implements Literal = StringModifier? MultiLineStringToken; + StringsLiteralPython implements Literal <200> = (StringLiteralPython | StringLiteral | MultiLineStringLiteral | MultiLineFStringLiteral | FStringPython)+; + FStringPython = (FSQStringPython | FDQStringPython | ("f"|"F") Char); + StringModifier = /*{cmpTokenRegEx(1, "(r|b|u|R|B|U)+")}?*/ type:Name; - // boolean literals for python - BooleanLiteralPython implements Literal, SignedLiteral = - source:["True" | "False"]; + //boolean literals for python + @Override + BooleanLiteral implements Literal, SignedLiteral = ource:["True" | "False"]; - // https://docs.python.org/dev/library/constants.html#Ellipsis - EllipsisLiteral implements Literal = "..."; - splittoken "..."; + //https://docs.python.org/dev/library/constants.html#Ellipsis + EllipsisLiteral implements Literal = "..."; + splittoken "..."; /*====================================== Statements ======================================*/ interface Statement; - // The StatementBlock represents a number of statements with the same indentation - scope StatementBlock = (BLOCK_START StatementBlockBody BLOCK_END) | Statement; - StatementBlockBody = Statement+; - - LiteralStatement implements ClassStatement = Literal STATEMENT_END; - - PassStatement implements Statement, ClassStatement = "pass" STATEMENT_END; - - ElseStatementPart = "else" ":" StatementBlock; - - PyQualifiedName = (Name || ".")+; - astrule PyQualifiedName = method public String joined(){ - return String.join(".", getNameList()); - }; - - // import statement - ImportStatement implements Statement = - ("from" {!next("import")}? (leadingDots:"."*) (module:PyQualifiedName)?)? - "import" - ( - ModuleList - | paren:"(" ModuleList ")" - ) - STATEMENT_END; - - ModuleList = star:"*" | ((ModuleWithOptionalAlias || ",")+ ","?); - - ModuleWithOptionalAlias = name:PyQualifiedName ("as" alias:Name)?; - - - // if-else statement - IfStatement implements Statement = "if" condition:Expression ":" thenStatement:StatementBlock - ("elif" elifCondition:Expression ":" elifStatement:StatementBlock )* - ElseStatementPart?; - - // assert statement - AssertStatement implements Statement = "assert" condition:Expression ("," errorMessage:Expression)? STATEMENT_END; - - // for statement - scope ForStatement implements Statement = async:"async"? "for" ForControl ":" StatementBlock - ElseStatementPart? ; - - ForControl = ForDecomposition "in" ForIterable; - interface ForDecomposition; - ForVariable implements Variable, ForDecomposition = Name; - ForDecompositionComma implements ForDecomposition = ForDecomposition "," (ForDecomposition ","?)?; - ForDecompositionParenthesis implements ForDecomposition = "(" ForDecomposition ")"; - - ForIterable = Expression; - - // while statement - WhileStatement implements Statement = "while" condition:Expression ":" StatementBlock - ElseStatementPart? ; - - BreakStatement implements Statement = "break" STATEMENT_END; - ContinueStatement implements Statement = "continue" STATEMENT_END; - - // try-except-finally statement - TryExceptStatement implements Statement = "try" ":" tryStatement:StatementBlock - ( + //The StatementBlock represents a number of statements with the same indentation + + scope StatementBlock = (BLOCK_START StatementBlockBody BLOCK_END) | Statement; + StatementBlockBody = Statement+; + + //Various statements + + //Helper-definition for qualified names in python. + PyQualifiedName = (Name || ".")+; + astrule PyQualifiedName = method public String joined(){ + return String.join(".", getNameList()); + }; + + //import statement + ImportStatement implements Statement ,ClassStatement = + ("from" {!next("import")}? (leadingDots:"."*) (module:PyQualifiedName)?)? + "import" + ( + ModuleList + | paren:"(" ModuleList ")" + ) + STATEMENT_END; + ModuleList = star:"*" | ((ModuleWithOptionalAlias || ",")+ ","?); + ModuleWithOptionalAlias = name:PyQualifiedName ("as" alias:Name)?; + + //Conditional statements + IfStatement implements Statement,ClassStatement = "if" condition:Expression ":" thenStatement:StatementBlock + ("elif" elifCondition:Expression ":" elifStatement:StatementBlock )* + ElseStatementPart?; + ElseStatementPart = "else" ":" StatementBlock; + ConditionalExecutionStatement implements Statement ,ClassStatement= "if" condition: Expression ":" Expression; + + //Assert statement + AssertStatement implements Statement,ClassStatement = "assert" condition:Expression ("," errorMessage:Expression)? STATEMENT_END; + + //for statement + scope ForStatement implements Statement,ClassStatement = async:"async"? "for" ForControl ":" StatementBlock + ElseStatementPart? ; + ForControl = ForDecomposition "in" ForIterable; + interface ForDecomposition; + ForVariable implements Variable, ForDecomposition = Name; + ForDecompositionComma implements ForDecomposition = ForDecomposition "," (ForDecomposition ","?)?; + ForDecompositionParenthesis implements ForDecomposition = "(" ForDecomposition ")"; + ForDecompositionBrackets implements ForDecomposition = "[" ForDecomposition "]"; + ForStarredVariable implements ForDecomposition = "*" ForDecomposition; + ForIterable = Expression; + + //while statement + WhileStatement implements Statement,ClassStatement = "while" condition:Expression ":" StatementBlock + ElseStatementPart? ; + + //Control-flow statements + PassStatement implements Statement, ClassStatement = "pass" STATEMENT_END; + BreakStatement implements Statement = "break" STATEMENT_END; + ContinueStatement implements Statement = "continue" STATEMENT_END; + + //try-except-finally statement + TryExceptStatement implements Statement,ClassStatement = "try" ":" tryStatement:StatementBlock ( - ExceptStatement+ - ElseStatementPart? - FinallyStatement? - ) - | FinallyStatement - ) - ; - - ExceptStatement = "except" (PyQualifiedName | "(" Alternatives ")" | (PyQualifiedName || "or")+ )? ("as" alias:Name)? ":" StatementBlock; - Alternatives = (PyQualifiedName || ",")+ | (PyQualifiedName || "or")+ | (PyQualifiedName || "|")+ ; - FinallyStatement = "finally" ":" finallyStatement:StatementBlock; - - // with open file statement - scope WithStatement implements Statement = async:"async"? "with" (WithStatementContents || ",")+ ":" - StatementBlock ; - - WithStatementContents = Expression ("as" target:Name)? ; // target has slicing/etc - - // variable declaration statement - LocalVariableDeclarationStatement implements Statement = VariableDeclaration STATEMENT_END; - - VariableDeclaration implements Variable = Name ( - ((":" TypeAnnotation)? "=" VariableInit) - | - (":" TypeAnnotation) - ); - - GlobalVariableDeclaration implements Statement = "global" (Name || ",")+ STATEMENT_END; - NonLocalVariableDeclaration implements Statement = "nonlocal" (Name || ",")+ STATEMENT_END; - MultiVariableDeclaration implements Statement = (Name || ",")+ ","? "=" Expression STATEMENT_END; - ParenMultiVariableDeclaration implements Statement = "(" (Name || ",")+ ","? ")" "=" Expression STATEMENT_END; - - interface VariableInit ; - - SimpleInit implements VariableInit = Expression ; - - // function declaration statement - interface FunctionDeclaration extends Function = Name ; - - SimpleFunctionDeclaration implements FunctionDeclaration, Statement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" FunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" + ( + ExceptStatement+ + ElseStatementPart? + FinallyStatement? + ) + | FinallyStatement + ); + scope ExceptStatement = "except" (PyQualifiedName | "(" Alternatives ")" | (PyQualifiedName || "or")+ )? ("as" alias:Alias)? ":" StatementBlock; + Alternatives = (PyQualifiedName || ",")+ | (PyQualifiedName || "or")+ | (PyQualifiedName || "or")+ ; + FinallyStatement = "finally" ":" finallyStatement:StatementBlock; + + //With open file statement + scope WithStatement implements Statement,ClassStatement = async:"async"? "with" (WithStatementContents || ",")+ ":" StatementBlock ; + WithStatementContents = Expression ("as" target:Name)? ; // target has slicing/etc + + //Variable related statements and definitions + LocalVariableDeclarationStatement implements Statement = VariableDeclaration STATEMENT_END; + VariableDeclaration implements Variable = Name ( + ((":" TypeAnnotation)? "=" VariableInit) + | + (":" TypeAnnotation) + ); + + GlobalVariableDeclaration implements Statement,ClassStatement = "global" names:(Name || ",")+ STATEMENT_END; + NonLocalVariableDeclaration implements Statement = "nonlocal" names:(Name || ",")+ STATEMENT_END; + MultiVariableDeclaration implements Statement,ClassStatement = (Name || ",")+ ","? "=" (Expression || "," )+ ","? STATEMENT_END; + ParenMultiVariableDeclaration implements Statement,ClassStatement = "(" (Name || ",")+ ","? ")" "=" (Expression || "," )+ ","? STATEMENT_END; + interface VariableInit ; + SimpleInit implements VariableInit = Expression ; + + //function declaration statement and function argument definitions + interface FunctionDeclaration extends Function = Name ; + SimpleFunctionDeclaration implements FunctionDeclaration, Statement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" FunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock ; - FunctionParameters = (FunctionParameter || ",")* ","?; - - interface FunctionParameter; - SimpleFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)?; - OptionalFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)? "=" Expression ; - VarArgFunctionParameter implements FunctionParameter, Variable = "*" Name GenericsAnnotation? (":" TypeAnnotation)?; - KWArgFunctionParameter implements FunctionParameter, Variable = "**" Name GenericsAnnotation? (":" TypeAnnotation)?; - StarFunctionParameter implements FunctionParameter = "*"; - - @Override - Arguments = "(" - (Argument || ",")* - ","? - ")"; - - interface Argument; - NormalArgument implements Argument = Expression; - NamedArgument implements Argument = paramName:Name "=" Expression; - - PyDecorator = "@" Expression STATEMENT_END; - - ReturnStatement implements Statement = "return" (Expression || ",")* ","? STATEMENT_END; - YieldStatement implements Statement = "yield" (Expression || ",")* ","? STATEMENT_END; - YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; - RaiseStatement implements Statement = "raise" (Expression ("from" Name)?)? STATEMENT_END; - - // Type related statemets - TypeAnnotationStatement implements Statement = Expression ":" TypeAnnotation STATEMENT_END; - TypeAliasStatement implements Statement = key("type") Name "=" Expression STATEMENT_END; - - ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; - - EmptyStatement implements Statement, ClassStatement = STATEMENT_END; - - MatchStatement implements Statement = key("match") (Expression || ",")+ ":" MatchBlock; - - scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; - CaseStatement = key("case") (Expression || ",")+ ("if" condition:Expression)? ":" StatementBlock; - - ConditionalExecutionStatement implements Statement = "if" condition: Expression ":" Expression; - - DeleteStatement implements Statement = "del" (Expression || ",")+ ","? STATEMENT_END; + FunctionParameters = (FunctionParameter || ",")* ","?; + interface FunctionParameter; + SimpleFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)?; + OptionalFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)? "=" Expression ; + VarArgFunctionParameter implements FunctionParameter, Variable = "*" Name GenericsAnnotation? (":" TypeAnnotation)?; + KWArgFunctionParameter implements FunctionParameter, Variable = "**" Name GenericsAnnotation? (":" TypeAnnotation)?; + StarFunctionParameter implements FunctionParameter = "*"; + @Override + Arguments = "(" + (Argument || ",")* + ","? + ")"; + interface Argument; + NormalArgument implements Argument = Expression; + NamedArgument implements Argument = paramName:Name "=" Expression; + + //Behaviour statements + ReturnStatement implements Statement = "return" (Expression || ",")* ","? STATEMENT_END; + YieldStatement implements Statement = "yield" (Expression || ",")* ","? STATEMENT_END; + YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; + RaiseStatement implements Statement,ClassStatement = "raise" (Expression ("from" Name)?)? STATEMENT_END; + + //Typing related statements + TypeDeclaration implements Statement = Expression ":" TypeAnnotation STATEMENT_END; + symbol TypeRuleStatement implements Statement,ClassStatement = key("type") Name "=" Expression STATEMENT_END; + ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; + + //Match statement + MatchStatement implements Statement,ClassStatement = key("match") (Expression || ",")+ ":" MatchBlock; + scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; + scope CaseStatement = key("case") (Expression || ",")+ ("if" condition:Expression)? ":" StatementBlock; + + //Another Statements + DeleteStatement implements Statement,ClassStatement= "del" (Expression || ",")+ ","? STATEMENT_END; + LiteralStatement implements ClassStatement = Literal STATEMENT_END; + EmptyStatement implements Statement, ClassStatement = STATEMENT_END; + PyDecorator = "@" Expression STATEMENT_END; /*====================================== Expressions ======================================*/ - // Aliased Expression only allowed in Case Statements (Checked with CoCo). - AliasedExpression implements Expression = Expression "as" alias:Name; - - ParenthesisExpression implements Expression = "(" Expression ")"; - - SpreadListExpression implements Expression = "*" Expression; - SpreadMappingExpression implements Expression = "**" Expression; - splittoken "**"; - - // ternary-operator expression - TernaryOperatorExpression implements Expression <200> = thenExpression:Expression ( "if" condition:Expression - "else" elseExpression:Expression )+ ; - - //mathematical expression - IntegerDivisionExpression implements Expression <165>, InfixExpression = left:Expression operator:"//" right:Expression ; - IntegerPowExpression implements Expression <195>, InfixExpression = left:Expression operator:"**" right:Expression ; - MatrixMultiplicationExpression implements Expression <200>, InfixExpression = left:Expression operator:"@" right:Expression; - - //logical expressions - AndExpression implements Expression <120>, InfixExpression = left:Expression operator:"and" right:Expression ; - OrExpression implements Expression <117>, InfixExpression = left:Expression operator:"or" right:Expression ; - NotExpression implements Expression <10> = "not" Expression ; - IsExpression implements Expression <130>, InfixExpression = left:Expression operator:"is" right:Expression ; - InExpression implements Expression <195>, InfixExpression = left:Expression operator:"in" right:Expression ; - NotInExpression implements Expression <195>, InfixExpression = left:Expression operator:"not" "in" right:Expression; // TODO: set operator to "not in" programmatically - - //Bitwise expressions - BitwiseAndExpression implements Expression <120>, InfixExpression = left:Expression operator:"&" right:Expression; - BitwiseOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"|" right:Expression; - BitwiseXOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"^" right:Expression; - BitwiseLeftShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:"<<" right:Expression; - BitwiseRightShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:">>" right:Expression; - - BitwiseOnesComplimentExpression implements Expression <120> = "~" Expression; - - // lambda expression - scope LambdaExpression implements Expression = "lambda" FunctionParameters ":" Expression ; - AppliedLambdaExpression implements Expression = "(" LambdaExpression ")" "(" Expression ")" ; - - AwaitExpression implements Expression = "await" Expression; - - AnnotatedAssignmentExpression implements Expression <60> = - left:Expression - ":" annotated: TypeAnnotation - "=" - right:Expression ","?; - - @Override - AssignmentExpression implements Expression <60> = - left:Expression - operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" - | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] - right:Expression ","?; - - UnpackingAssignmentExpression implements Expression <60> = - "(" left:Expression ("," left:Expression)* ","? ")" "=" - right: Expression; - - // 6.3.3 - Slicing - IndexExpression implements Expression = Expression "[" (IndexExpressionInner || ",")+ tuple:","? "]"; - - // slice_item - interface IndexExpressionInner; - - SimpleIndex implements IndexExpressionInner = Expression; - ProperSlice implements IndexExpressionInner = lower:Expression? ":" upper:Expression? (":" stride:Expression?)?; - - // Walrus operator - PyAssignmentExpression implements Expression = variable:Name operator:":=" right:Expression; - - // List/Set/Dict comprehension - ListComprehensionExpression implements Expression = "[" Expression "for" ForControl GeneratorFilter? "]"; - SetComprehensionExpression implements Expression = "{" Expression "for" ForControl GeneratorFilter? "}"; - DictComprehensionExpression implements Expression = "{" Name ":" Expression "for" ForControl GeneratorFilter? "}"; - GeneratorExpression implements Expression = Expression "for" ForControl GeneratorFilter? ; - - GeneratorFilter = "if" condition:Expression; + //Aliased Expression only allowed in Case Statements (Checked with CoCo). + AliasedExpression implements Expression = Expression "as" alias:Alias; + symbol Alias = Name; + //Spreadlist expression + SpreadListExpression implements Expression = "*" Expression; + SpreadMappingExpression implements Expression = "**" Expression; + splittoken "**"; + + // ternary-operator expression + TernaryOperatorExpression implements Expression <200> = thenExpression:Expression ( "if" condition:Expression + "else" elseExpression:Expression )+ ; + + //mathematical expression + IntegerDivisionExpression implements Expression <165>, InfixExpression = left:Expression operator:"//" right:Expression ; + IntegerPowExpression implements Expression <195>, InfixExpression = left:Expression operator:"**" right:Expression ; + MatrixMultiplicationExpression implements Expression <200>, InfixExpression = left:Expression operator:"@" right:Expression; + + //logical expressions + AndExpression implements Expression <120>, InfixExpression = left:Expression operator:"and" right:Expression; + OrExpression implements Expression <117>, InfixExpression = left:Expression operator:"or" right:Expression; + NotExpression implements Expression <10> = "not" Expression ; + IsExpression implements Expression <130>, InfixExpression = left:Expression operator:"is" right:Expression; + InExpression implements Expression <195>, InfixExpression = left:Expression operator:"in" right:Expression; + NotInExpression implements Expression <195>, InfixExpression = left:Expression operator:"not" "in" right:Expression; // TODO: set operator to "not in" programmatically + + //Bitwise expressions + BitwiseAndExpression implements Expression <120>, InfixExpression = left:Expression operator:"&" right:Expression; + BitwiseOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"|" right:Expression; + BitwiseXOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"^" right:Expression; + BitwiseLeftShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:"<<" right:Expression; + BitwiseRightShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:">>" right:Expression; + BitwiseOnesComplimentExpression implements Expression <120> = "~" Expression; + + // lambda expression + scope LambdaExpression implements Expression = "lambda" FunctionParameters ":" Expression; + AppliedLambdaExpression implements Expression = "(" LambdaExpression ")" "(" Expression ")"; + + //Assignment expressions + AnnotatedAssignmentExpression implements Expression <60> = + left:Expression + ":" annotated: TypeAnnotation + "=" right:Expression ","?; + @Override + AssignmentExpression implements Expression <60> = + left:Expression + operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" + | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] + right:Expression ","?; + UnpackingAssignmentExpression implements Expression <60> = + "(" left:Expression ("," left:Expression)* ","? ")" "=" right: Expression; + + // 6.3.3 - Slicing + IndexExpression implements Expression = Expression "[" (IndexExpressionInner || ",")+ tuple:","? "]"; + + // slice_item + interface IndexExpressionInner; + SimpleIndex implements IndexExpressionInner = Expression; + ProperSlice implements IndexExpressionInner = lower:Expression? ":" upper:Expression? (":" stride:Expression?)?; + + // Walrus operator + PyAssignmentExpression implements Expression = variable:Name operator:":=" right:Expression; + + //Await expression + AwaitExpression implements Expression = "await" Expression; + + //List/Set/Dict comprehension + ListComprehensionExpression implements Expression = "[" Expression "for" ForControl GeneratorFilter* "]"; + SetComprehensionExpression implements Expression = "{" Expression "for" ForControl GeneratorFilter* "}"; + DictComprehensionExpression implements Expression = "{" Name ":" Expression "for" ForControl GeneratorFilter* "}"; + GeneratorExpression implements Expression = Expression "for" ForControl GeneratorFilter? ; + GeneratorFilter = "if" condition:Expression; /*===========================Classes======================================*/ - // class symbol - interface scope symbol PythonClass = Name ; + // class symbol + interface scope symbol PythonClass = Name ; - // class declaration statement - ClassDeclaration implements PythonClass, Statement, ClassStatement = PyDecorator* "class" Name GenericsAnnotation? ( "(" ((superClasses:PyQualifiedName TypeAnnotation? | arguments:NamedArgument) ","?)* ")" )? ":" ClassStatementBlock ; - interface ClassStatement; + // class declaration statement + ClassDeclaration implements PythonClass, Statement, ClassStatement = PyDecorator* "class" Name GenericsAnnotation? + ( "(" ((superClasses:PyQualifiedName TypeAnnotation? | arguments:NamedArgument) ","?)* ")" )? ":" ClassStatementBlock ; - ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement; - ClassStatementBlockBody = ClassStatement+; + interface ClassStatement; + ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement; + ClassStatementBlockBody = ClassStatement+; + ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? + "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; - ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; - - ClassFunctionParameters = (FunctionParameter || ",")* ","?; - - ClassAttributes implements ClassStatement = VariableDeclaration STATEMENT_END; - - ClassCommentStatement implements ClassStatement = MultiLineStringLiteral STATEMENT_END; + ClassFunctionParameters = (FunctionParameter || ",")* ","?; + ClassAttributes implements ClassStatement = VariableDeclaration STATEMENT_END; + ClassCommentStatement implements ClassStatement = MultiLineStringLiteral STATEMENT_END; /*====================================== Type Annotations ======================================*/ - interface TypeAnnotation; - GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation; - StringTypeAnnotation implements TypeAnnotation = StringLiteralPython; - TupleTypeAnnotation implements TypeAnnotation = "(" (TypeAnnotation || ",")+ ","? ")"; - QualifiedTypeAnnotation implements TypeAnnotation = type:PyQualifiedName ("[" typeParams:(TypeAnnotation || ",")* "]")?; - AlternativeTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "|" rhs:TypeAnnotation; - CommaTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "," rhs:TypeAnnotation ","?; - ListTypeAnnotation implements TypeAnnotation = "[" TypeAnnotation "]"; - EllipsisTypeAnnotation implements TypeAnnotation = "..."; - ComplexTypeAnnotation implements TypeAnnotation <200> = Expression; + interface TypeAnnotation; + GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation; + StringTypeAnnotation implements TypeAnnotation = StringLiteralPython; + TupleTypeAnnotation implements TypeAnnotation = "(" (TypeAnnotation || ",")+ ","? ")"; + QualifiedTypeAnnotation implements TypeAnnotation = type:PyQualifiedName ("[" typeParams:(TypeAnnotation || ",")* "]")?; + AlternativeTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "|" rhs:TypeAnnotation; + CommaTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "," rhs:TypeAnnotation ","?; + ListTypeAnnotation implements TypeAnnotation = "[" TypeAnnotation "]"; + EllipsisTypeAnnotation implements TypeAnnotation = "..."; + ComplexTypeAnnotation implements TypeAnnotation <200> = Expression; /*====================================== Generics ======================================*/ - GenericsAnnotation = "[" Generics? "]"; - Generics = (Generic || ",")+ ; - Generic = PyQualifiedName (":" TypeAnnotation)?; - -} + GenericsAnnotation = "[" Generics? "]"; + Generics = (Generic || ",")+ ; + Generic = PyQualifiedName (":" TypeAnnotation)?; +} \ No newline at end of file From 0f6f43ae892d1e3f5b8021515d46cb012d764469 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Sun, 3 May 2026 14:21:09 +0200 Subject: [PATCH 13/26] Changing name of a Nonterminal --- src/main/grammars/de/monticore/Python.mc4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index a458079..3dae04b 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -81,8 +81,8 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; PyFloatLiteral implements NumericLiteral <200> = PyFloat; - token PyImaginaryNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; - PyImaginaryNumberLiteral implements NumericLiteral <95> = PyImaginaryNumber; + token PyComplexNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; + PyComplexNumberLiteral implements NumericLiteral <95> = PyComplexNumber; //PEP 515 token DigitsPart = Digit ('_'? Digit)*; From 0a507a54579b6c6cf9bc1aba4bdfe382db7a84d3 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Sun, 3 May 2026 15:01:36 +0200 Subject: [PATCH 14/26] Allowing nonlocal as class-statement --- src/main/grammars/de/monticore/Python.mc4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 3dae04b..96a5c6f 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -214,7 +214,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ); GlobalVariableDeclaration implements Statement,ClassStatement = "global" names:(Name || ",")+ STATEMENT_END; - NonLocalVariableDeclaration implements Statement = "nonlocal" names:(Name || ",")+ STATEMENT_END; + NonLocalVariableDeclaration implements Statement,ClassStatement = "nonlocal" names:(Name || ",")+ STATEMENT_END; MultiVariableDeclaration implements Statement,ClassStatement = (Name || ",")+ ","? "=" (Expression || "," )+ ","? STATEMENT_END; ParenMultiVariableDeclaration implements Statement,ClassStatement = "(" (Name || ",")+ ","? ")" "=" (Expression || "," )+ ","? STATEMENT_END; interface VariableInit ; From f7004908cd017017235ec1bc8a84adc541a02c2c Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Sun, 3 May 2026 18:35:44 +0200 Subject: [PATCH 15/26] Fixing Case and Except Statements, Allowing Generics in Typedeclarations --- src/main/grammars/de/monticore/Python.mc4 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 96a5c6f..09e4882 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -197,7 +197,8 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ) | FinallyStatement ); - scope ExceptStatement = "except" (PyQualifiedName | "(" Alternatives ")" | (PyQualifiedName || "or")+ )? ("as" alias:Alias)? ":" StatementBlock; + scope ExceptStatement = "except" (Expression | Expression ("as" alias:Alias)?)? ":" ExceptStatementBlock; + ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; Alternatives = (PyQualifiedName || ",")+ | (PyQualifiedName || "or")+ | (PyQualifiedName || "or")+ ; FinallyStatement = "finally" ":" finallyStatement:StatementBlock; @@ -243,18 +244,19 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit //Behaviour statements ReturnStatement implements Statement = "return" (Expression || ",")* ","? STATEMENT_END; YieldStatement implements Statement = "yield" (Expression || ",")* ","? STATEMENT_END; - YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; + YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; //PEP380 RaiseStatement implements Statement,ClassStatement = "raise" (Expression ("from" Name)?)? STATEMENT_END; //Typing related statements TypeDeclaration implements Statement = Expression ":" TypeAnnotation STATEMENT_END; - symbol TypeRuleStatement implements Statement,ClassStatement = key("type") Name "=" Expression STATEMENT_END; + symbol TypeRuleStatement implements Statement,ClassStatement = key("type") Name GenericsAnnotation? "=" Expression STATEMENT_END; ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; //Match statement - MatchStatement implements Statement,ClassStatement = key("match") (Expression || ",")+ ":" MatchBlock; + MatchStatement implements Statement,ClassStatement = key("match") Expression ":" MatchBlock; scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; - scope CaseStatement = key("case") (Expression || ",")+ ("if" condition:Expression)? ":" StatementBlock; + scope CaseStatement = key("case") (Expression || "|")+ ("as" Alias)? ("if" condition:Expression)? ":" CaseStatementBlock; + CaseStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; //Another Statements DeleteStatement implements Statement,ClassStatement= "del" (Expression || ",")+ ","? STATEMENT_END; @@ -373,5 +375,5 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit GenericsAnnotation = "[" Generics? "]"; Generics = (Generic || ",")+ ; - Generic = PyQualifiedName (":" TypeAnnotation)?; + Generic = TypeAnnotation (":" TypeAnnotation)?; } \ No newline at end of file From 4786bb4c67c8e378b3cf0c73e26fd8fb906ae0f8 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Mon, 4 May 2026 17:56:03 +0200 Subject: [PATCH 16/26] Rework of ForControl closer to what is allowed by python --- src/main/grammars/de/monticore/Python.mc4 | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 09e4882..1ce3d4e 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -169,13 +169,14 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit //for statement scope ForStatement implements Statement,ClassStatement = async:"async"? "for" ForControl ":" StatementBlock ElseStatementPart? ; - ForControl = ForDecomposition "in" ForIterable; + ForControl = ForList "in" ForIterable; interface ForDecomposition; + ForList = (ForDecomposition || ",")+ ","?; ForVariable implements Variable, ForDecomposition = Name; - ForDecompositionComma implements ForDecomposition = ForDecomposition "," (ForDecomposition ","?)?; - ForDecompositionParenthesis implements ForDecomposition = "(" ForDecomposition ")"; - ForDecompositionBrackets implements ForDecomposition = "[" ForDecomposition "]"; - ForStarredVariable implements ForDecomposition = "*" ForDecomposition; + ForDecompositionParenthesis implements ForDecomposition = "(" ForList? ")"; + ForDecompositionBrackets implements ForDecomposition = "[" ForList? "]"; + ForStarredVariable implements ForDecomposition = "*" ForDecomposition ; + ForPyQualifiedName implements ForDecomposition = PyQualifiedName; ForIterable = Expression; //while statement From 94ea3c7441c7716f59d82a2a22218d566cab30e8 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Mon, 4 May 2026 19:25:32 +0200 Subject: [PATCH 17/26] Incooperating PEP758 Removing test that became wrong --- src/main/grammars/de/monticore/Python.mc4 | 10 +++++++--- src/test/java/de/monticore/python/PythonTest.java | 9 --------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 1ce3d4e..492854c 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -198,11 +198,15 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit ) | FinallyStatement ); - scope ExceptStatement = "except" (Expression | Expression ("as" alias:Alias)?)? ":" ExceptStatementBlock; + scope ExceptStatement = "except" ExceptPattern? ":" ExceptStatementBlock; ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; - Alternatives = (PyQualifiedName || ",")+ | (PyQualifiedName || "or")+ | (PyQualifiedName || "or")+ ; FinallyStatement = "finally" ":" finallyStatement:StatementBlock; - + //PEP + interface ExceptPattern; + ExpressionListing implements ExceptPattern = (Expression || ",")+; + ParenthesisedExpressionListing implements ExceptPattern = "(" (Expression || ",")+ ")" ("as" Alias)?; + StarredExpressionListing implements ExceptPattern = "*"(Expression || ",")+; + StarredParenthesisedExpressionListing implements ExceptPattern = "*""(" (Expression || ",")+ ")" ("as" Alias)?; //With open file statement scope WithStatement implements Statement,ClassStatement = async:"async"? "with" (WithStatementContents || ",")+ ":" StatementBlock ; WithStatementContents = Expression ("as" target:Name)? ; // target has slicing/etc diff --git a/src/test/java/de/monticore/python/PythonTest.java b/src/test/java/de/monticore/python/PythonTest.java index 904b01d..1ee7bc1 100644 --- a/src/test/java/de/monticore/python/PythonTest.java +++ b/src/test/java/de/monticore/python/PythonTest.java @@ -638,15 +638,6 @@ public void parseInvalidClassDeclaration() { " e.count+=1\n" + " self.list_x.append(i)\n" ); - //for loop in class - parseModelFromStringAndExpectFail( - "class myClass:\n" + - " def function_name(x,y):\n" + - " print(x,y)\n" + - " for i in range(4):\n" + - " print(i)\n" - ); - } /*===========================Other======================================*/ From 08c5107d115e242a27134af10cafe53bb59e3688 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Mon, 4 May 2026 19:26:27 +0200 Subject: [PATCH 18/26] Incooperating PEP758 comment --- src/main/grammars/de/monticore/Python.mc4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 492854c..1c652d9 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -201,7 +201,7 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit scope ExceptStatement = "except" ExceptPattern? ":" ExceptStatementBlock; ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; FinallyStatement = "finally" ":" finallyStatement:StatementBlock; - //PEP + //PEP758 interface ExceptPattern; ExpressionListing implements ExceptPattern = (Expression || ",")+; ParenthesisedExpressionListing implements ExceptPattern = "(" (Expression || ",")+ ")" ("as" Alias)?; From 78659912ca6455b515f1514dcaefbc887ffa0133 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 6 May 2026 16:02:46 +0200 Subject: [PATCH 19/26] Refining Tests --- .../java/de/monticore/python/PythonTest.java | 94 ++++++++++++++++++- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/src/test/java/de/monticore/python/PythonTest.java b/src/test/java/de/monticore/python/PythonTest.java index 1ee7bc1..07883bb 100644 --- a/src/test/java/de/monticore/python/PythonTest.java +++ b/src/test/java/de/monticore/python/PythonTest.java @@ -40,6 +40,8 @@ public void parseFloorDiv(){ @Test public void parseListComprehension(){ parseModelFromStringAndExpectSuccess("rgb = [i for i in range(0, hlen, hlen // 3)]\n"); + parseModelFromStringAndExpectSuccess("rgb = [i for i, *b in range(0, hlen, hlen // 3) if cond1 if cond2]\n"); + } @Test @@ -231,7 +233,31 @@ public void parseInvalidForLoopStatement() { " print(x)\n" ); } - + @Test + public void parseValidMatchStatement(){ + parseModelFromStringAndExpectSuccess( + "match x:\n" + + " case (a,b as var1) as var2: pass\n" + + " case (a,b,c) as var3: pass\n" + ); + parseModelFromStringAndExpectSuccess( + "match x:\n" + + " case a as var1: pass\n" + + " case (b,c) as var2: pass\n" + ); + } + @Test + public void parseInvalidMatchStatement(){ + parseModelFromStringAndExpectFail( + "match x:\n" + + " case (a,b as var1 ) as var2: pass\n" + + " case as var3: pass\n" + ); + parseModelFromStringAndExpectFail( + "match x:\n" + + " case: pass\n" + ); + } //valid while statements @Test public void parseValidWhileLoopStatement() { @@ -326,7 +352,13 @@ public void parseValidTryExceptStatements() { "finally:\n" + " print(\"Done\")\n" ); - + //Additional test inspired by https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py + parseModelFromStringAndExpectSuccess( + "try:\n" + + " i = 1//0\n" + + "except ZeroDivisionError or CustomZeroDivisionError as exp:\n" + + " print(\"Can not divide by zero\")\n" + ); } //invalid try-except-finally statements @@ -349,7 +381,15 @@ public void parseInvalidTryExceptStatements() { "else:\n" + " print(\"Success\")\n" ); - + // missing aliased expression + parseModelFromStringAndExpectFail( + "try:\n" + + " i = 1//0\n" + + "except as exp:\n" + + " print(\"Can not divide by zero\")\n" + + "else:\n" + + " print(\"Success\")\n" + ); // duplicate finally parseModelFromStringAndExpectFail( "try:\n" + @@ -489,6 +529,12 @@ public void parseValidLambdaStatement() { parseModelFromStringAndExpectSuccess("lambda: 1\n"); parseModelFromStringAndExpectSuccess("lambda x: x\n"); parseModelFromStringAndExpectSuccess("lambda x, y: x + y\n"); + //Additional tests inspired by https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py + parseModelFromStringAndExpectSuccess("lambda *a : 1\n"); + parseModelFromStringAndExpectSuccess("lambda **a : 1\n"); + parseModelFromStringAndExpectSuccess("lambda *a, **b : 1\n"); + parseModelFromStringAndExpectSuccess("lambda a=name : 1\n"); + parseModelFromStringAndExpectSuccess("lambda a=name, b= q+1 : 1\n"); } //invalid lambda statement @@ -499,20 +545,33 @@ public void parseInvalidLambdaStatement() { parseModelFromStringAndExpectFail("lambda x, y z\n"); } - // valid lambda statement + // valid raise statement @Test public void parseValidRaiseStatement() { parseModelFromStringAndExpectSuccess("raise RuntimeError('Error')\n"); parseModelFromStringAndExpectSuccess("raise\n"); } - //invalid lambda statement + //invalid raise statement @Test public void parseInvalidRaiseStatement() { parseModelFromStringAndExpectFail("raise RuntimeError('Error'), ArithmeticError('Error')\n"); parseModelFromStringAndExpectFail("raise RuntimeError('Error') ArithmeticError('Error')\n"); } + // tests for yield inspired by https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py + // valid yield statement + @Test + public void parseValidYieldStatement() { + parseModelFromStringAndExpectSuccess("def function():\n i+=1 \n yield i\n"); + parseModelFromStringAndExpectSuccess("def function():\n yield from anotherFunction()\n"); + } + //invalid yield statement + @Test + public void parseInvalidYieldStatement() { + parseModelFromStringAndExpectFail("def function():\n yield yield\n"); + parseModelFromStringAndExpectFail("def function():\n yield from\n"); + } /*===========================Literals======================================*/ // valid string literals python @@ -530,6 +589,21 @@ public void parseInvalidStringPython() { parseModelFromStringAndExpectFail("helloworld = Hello World\n"); } + //Same tests as above just with the modifier f (and with a single char string) + // valid fstring literals python + @Test + public void parseValidFStringPython() { + parseModelFromStringAndExpectSuccess("helloworld = f\" \"\n"); + parseModelFromStringAndExpectSuccess("helloworld = F'Hello World'\n"); + } + + // invalid fstring literals python + @Test + public void parseInvalidFStringPython() { + parseModelFromStringAndExpectFail("helloworld = f\"Hello World\n"); + parseModelFromStringAndExpectFail("helloworld = F'Hello World\n"); + parseModelFromStringAndExpectFail("helloworld = Hello World\n"); + } // boolean literals for python @Test public void parseValidBooleanPython() { @@ -564,6 +638,16 @@ public void parseInvalidTernaryOperator() { parseModelFromStringAndExpectFail("x = u if a==b else\n"); } + // tests for Walrus inspired by https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py + @Test + public void parseValidWalrusOperator(){ + parseModelFromStringAndExpectSuccess("def function():\n yield a:=2\n"); + } + @Test + public void parseInalidWalrusOperator(){ + parseModelFromStringAndExpectFail("def function():\n a:= Yield 2\n"); + } + // valid logical expressions @Test public void parseValidLogicalExpressions() { From 32b2e25a3287dbebd6749dc5cc45c1cb2f34ccb1 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 6 May 2026 17:32:11 +0200 Subject: [PATCH 20/26] preperation of pullrequest --- Change documentation/Pullreguest.md | 241 +++++++++++++++++++ Change documentation/knownToBeUnsupported.md | 32 +++ 2 files changed, 273 insertions(+) create mode 100644 Change documentation/Pullreguest.md create mode 100644 Change documentation/knownToBeUnsupported.md diff --git a/Change documentation/Pullreguest.md b/Change documentation/Pullreguest.md new file mode 100644 index 0000000..54a1951 --- /dev/null +++ b/Change documentation/Pullreguest.md @@ -0,0 +1,241 @@ +# Pullrequest to incorporate new Python functionality. +In this Markdown all changes will be presented and reasoned. +## Grammar + +### Rework of For Decomposition +Python currently allows another targets inside the for constructs, thereby we change our ForControl non terminal to be closer to the python language. + +Old: +``` + ForControl = ForDecomposition "in" ForIterable; + interface ForDecomposition; + ForDecompositionComma implements ForDecomposition = ForDecomposition "," (ForDecomposition ","?)?; + ForDecompositionParenthesis implements ForDecomposition = "("ForDecomposition ")"; + ForDecompositionBrackets implements ForDecomposition = "[" ForDecomposition "]"; + ForStarredVariable implements ForDecomposition = "*" ForDecomposition; +``` +New: +``` + ForControl = ForList "in" ForIterable; + interface ForDecomposition; + ForList = (ForDecomposition || ",")+ ","?; + ForDecompositionParenthesis implements ForDecomposition = "(" ForList? ")"; + ForDecompositionBrackets implements ForDecomposition = "[" ForList? "]"; + ForStarredVariable implements ForDecomposition = "*" ForDecomposition ; + ForPyQualifiedName implements ForDecomposition = PyQualifiedName; +``` +Python: ("[...]" Marks optional) +``` +comprehension: assignment_expression comp_for +comp_for: ["async"] "for" target_list "in" or_test [comp_iter] +target_list: target ("," target)* [","] +target: identifier + | "(" [target_list] ")" + | "[" [target_list] "]" + | attributeref + | subscription + | "*" target + +``` + Sources: https://docs.python.org/3/reference/simple_stmts.html#grammar-token-python-grammar-target_list and https://docs.python.org/3/reference/expressions.html (6.2.5) + +### Rework of Boolean +Instead of Defining a new BooleanLiteralPython, we override the Monticore BooleanLiteral to allow 'true' and 'false' as names, tests have shown that they can be used as such. + +### Rework of fstrings +It proved to be difficult to support all variants of fStrings, so far they were parsed as a singular string token, thereby the content of curly brackets was considered text, yet tests have shown that line breaks are allowed within curly brackets. Another issue is the allowed usage of the same quotes " or ' in the outer string as well as inside the curly brackets. Finally it is possible to add f to chars in python such as ```f'a'``` this was so far an issue as only strings were allowed to be modified and not chars. +With the following changes we aim combat these issues. + +1) We don't allow f and F as string modifier (line 121) +2) We add chars to the string definitions (line 115,line 120) +3) We add new tokens for fstrings + ``` + //Double and single quoted strings with an f modifier f'text1{exp}text2', separate definition to allow more expressions. + token FSQStringPython + = ('f'|'F') '\'' (StringFSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; + token FDQStringPython + = ('f'|'F') '"' (StringFDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; + fragment token StringFSQCharactersPython = (StringFSQCharacterPython)+; + fragment token StringFDQCharactersPython = (StringFDQCharacterPython)+; + fragment token StringFSQCharacterPython = ~ ('\''| '\\'| '{' )| PythonEscapeSequence |InnerFString; + fragment token StringFDQCharacterPython = ~ ('"'| '\\'| '{') | PythonEscapeSequence |InnerFString; + fragment token InnerFString = '{' ~('}')* '}'; + ``` +4) We similarly define fstring literals and non terminals for multilinestring tokens +5) We add a new FStringPython non terminal (line 120) and add it to the StringLiteralPython (line 112-117) + + +### Addition of complex numbers. +Python supports the usage of complex numbers, they are formatted as: + (real number)? (+|-)? (imaginaryNumber)'j' +A token PyComplexNumber was added supporting these numbers, additional a matching literal for their usage within python code. +``` token PyComplexNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; + PyComplexNumberLiteral implements NumericLiteral <95> = PyComplexNumber; +``` +Source for cmath: https://docs.python.org/3/library/cmath.html +### Addition of generics. + +We added support for generics as follows: +``` + GenericsAnnotation = "[" Generics? "]"; + Generics = (Generic || ",")+ ; + Generic = TypeAnnotation (":" TypeAnnotation)?; +``` +This is done similarly to https://docs.python.org/3/reference/compound_stmts.html#type-params. +But differently to use the existing TypeAnnotation non terminal. To support the functionality described in the source above we added "GenericsAnnotation?" to: + - The the non terminals implementting the interface FunctionParameter. + - ClassFunctionDeclaration,ClassDeclaration,SimpleFunctionDeclaration,TypeDeclarationStatement + and the TypeAnnotations ``` GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation``` + +### Statements + +### ClassStatements +Import, If, Assert, For, While, ConditionalExecution, With, GlobalVariableDeclaration, NonLocalVariableDeclaration, MultiVariableDeclaration, ParenMultiVariableDeclaration, Raise, TypeRuleStatement,Delete statements are now implementing the ClassStatement interface. By experimenting and testing they showed to be allowed inside classes. + +#### New Statements + +##### Nonlocal +For a detailed description of the nonlocal keyword see: https://docs.python.org/3/tutorial/classes.html under 9.2 . Summarizing the source, nonlocal allows a function or class defined inside another function to accesses the enclosing functions variables. Without the non local keyword these variables are read only to nested functions and classes. Our implementation is similar to pythons https://docs.python.org/3/reference/simple_stmts.html#nonlocal. + +Pythons: ```nonlocal_stmt: "nonlocal" identifier ("," identifier)* ```(see source above) +Ours : ```NonLocalVariableDeclaration implements Statement,ClassStatement = "nonlocal" names:(Name || ",")+ STATEMENT_END;``` + +##### Yield from +Yield from statements in the form: 'yield' 'from' expression have been added in https://peps.python.org/pep-0380/ ,we support them similarly to python https://docs.python.org/3/reference/expressions.html#grammar-token-python-grammar-yield_expression. + +Pythons: ```yield_from: "yield" "from" expression``` (see source above) +Ours:```YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; ``` + + +##### Type Declaration Statement +To allow type hints as ``` var : int``` without an assignment (in comparison to augmented assignments), we add the new TypeDeclartationStatement. +``` +TypeDeclaration implements Statement = Expression ":" TypeAnnotation STATEMENT_END; +``` +Python incorporates this functionality in the augmented assignments, yet this caused issues for us. see https://docs.python.org/3/reference/simple_stmts.html#index-15. + +##### Type Aliases +Python allows to declare type aliases. Where a type is aliased under an identifier, see https://docs.python.org/3/reference/simple_stmts.html#grammar-token-python-grammar-type_stmt. + +We implement the support similarly, also we define the type alias as symbol. +Python: ```type_stmt: 'type' identifier [type_params] "=" expression``` (see source above) +Ours: ```symbol TypeRuleStatement implements Statement,ClassStatement = key("type") Name GenericsAnnotation? "=" Expression STATEMENT_END;``` +'type' needs to be declared as local keyword as tests have shown that it can be used as variable name. Python also specifies this https://docs.python.org/3/reference/lexical_analysis.html#soft-keywords . + +#### Changes + +##### Global +So far we only allowed a single variable to be declared as global in a global statement. +But Python allows multiple variables to be declared global in the same statement separated by commata see: https://docs.python.org/3/reference/simple_stmts.html#global , also so far we allowed a type annotation within a global statement which is not allowed, thereby this support is removed. +Additionally global can be used in classes as described in the source above. + +Old: ```GlobalVariableDeclaration implements Statement = "global" Name (":" TypeAnnotation)? STATEMENT_END;``` +New:```GlobalVariableDeclaration implements Statement,ClassStatement = "global" (Name || ",")+ STATEMENT_END;``` + +##### Class Statement Block +A ClassStatementBlock is now allowed to be a singular ClassStatement to allow classes such as: +``` +class name1: pass +class name2: a=1 +``` +Old: ```ClassStatementBlock = BLOCK_START ClassStatementBlockBody BLOCK_END;``` +New: ```ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement;``` + +Test have shown that this is needed. + +#### Match Statement +Case statements in python allow patterns to be aliased (as example ```case Type as t:```) see https://docs.python.org/3/reference/compound_stmts.html#the-match-statement and https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-patterns. To allow this, case statements now span a scope to capture the alias. +Old: +``` +MatchStatement implements Statement = key("match") Expression ":" MatchBlock; +scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; +CaseStatement = key("case") (Expression || "|")+ ("if" condition:Expression)? ":" StatementBlock; +``` +New: +``` +MatchStatement implements Statement,ClassStatement = key("match") Expression ":" MatchBlock; +scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; +scope CaseStatement = key("case") (Expression || "|")+ ("as" Alias)? ("if" condition:Expression)? ":" CaseStatementBlock; +CaseStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; +``` +(Alias is defined as ```symbol Alias = Name;````) +Aditionally we allowed match statements inside classes. + +#### Try-Except +PEP758 Added support for leaving out parenthesis. Further so far the support for various Expressions and starred expressions were missing as specified in https://peps.python.org/pep-0758/ we added the support for this accordingly as specified. Also similarly to the new case statements, excepts now span a scope with the alias non terminal. +Old: +``` + ExceptStatement = "except" (PyQualifiedName? | "(" (PyQualifiedName || ",")+ ")") ("as" alias:Name)? ":" StatementBlock; + +``` +New: +``` +scope ExceptStatement = "except" ExceptPattern? ":" ExceptStatementBlock; +ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; +//PEP758 +interface ExceptPattern; +ExpressionListing implements ExceptPattern = (Expression || ",")+; +ParenthesisedExpressionListing implements ExceptPattern = "(" (Expression || ",")+ ")" ("as" Alias)?; +StarredExpressionListing implements ExceptPattern = "*"(Expression || ",")+; +StarredParenthesisedExpressionListing implements ExceptPattern = "*""(" (Expression || ",")+ ")" ("as" Alias)?; +``` +### Expressions + +#### Changes + +##### Assignment +As defined by https://docs.python.org/3/reference/simple_stmts.html#index-14 python's assignment statements additionally supports "//=" "@=" "**=" thereby we add this support by overriding the AssignmentExpression from Monticore by simply adding the three additional operators to the square brackets. +``` + @Override + AssignmentExpression implements Expression <60> = + left:Expression + operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" + | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] + right:Expression; + +``` +##### Augmented Assignment +So far we allowed the same operators as in the assignment expression, yet python only allows "=" see https://docs.python.org/3/reference/simple_stmts.html#index-15. + +Thereby we remove the square brackets and only allow "=". +Old: +``` + AnnotatedAssignmentExpression implements Expression <60> = + left:Expression + ":" annotated: TypeAnnotation + operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" + | "^=" | ">>=" | ">>>=" | "<<=" | "%=" ] + right:Expression; +``` +New: +``` + AnnotatedAssignmentExpression implements Expression <60> = + left:Expression + ":" annotated: TypeAnnotation + "=" + right:Expression; +``` +#### Comprehensions +We allowed multiple if inside a comprehension by changing ? to * at the Generator filters. +This is allowed by python as specified here: https://docs.python.org/3/reference/expressions.html#grammar-token-python-grammar-comp_for + +### Another +We added some trailing ","? to rules where they were noticed to be allowed while testing. We also formated the grammar and added some comments +## Code +In this section i will add the commits instead of the code snippets for readability reasons. +### Preprocessor +Commit: https://github.com/MontiCore/MCPython/commit/8c519eff8bcdf027ae421726fc36929f0bde2dd1 +In line 75-77 i added that the continue line token is always ignored to allow statements like: +``` + function(var1, var2,\ + var3, var4) +``` +as these could cause issues. +### Visitor and CoCos related to aliased expressions. +To avoid aliased expressions to be used anywhere we added a matching CoCo and Visitor. +Commit: https://github.com/MontiCore/MCPython/commit/ba9e8f18fb1c038a2cca33527a1e1bb8552778ab + +### Adaptation of new tests +We removed a test regarding invalid classes that are now valid (because for loops are now allowed as class statements). +We added new test cases for Python +...to do more specific... diff --git a/Change documentation/knownToBeUnsupported.md b/Change documentation/knownToBeUnsupported.md new file mode 100644 index 0000000..786c25d --- /dev/null +++ b/Change documentation/knownToBeUnsupported.md @@ -0,0 +1,32 @@ +# Currently known to be unsupported + +## Tuples without parentheses +The usage of tuples that are not parenthesized is not supported as these causes issues with antlr. (See https://www.geeksforgeeks.org/python/when-are-parentheses-required-around-a-tuple-in-python/ for unparenthesized tuples) + We define the rules: + + ```TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ;``` + ```SimpleInit implements VariableInit = Expression ;``` + and ExpressionBasis.mc4 ```LiteralExpression implements Expression <340> = Literal;``` + + Thereby without the parentheses we would cause left recursion which is also reachable by ``LiteralStatement implements ClassStatement = Literal STATEMENT_END;```. + +## Unicode names + Further for now only names with latin letters are permitted, in python another chars are allowed as well, see https://docs.python.org/3/reference/lexical_analysis.html#identifiers. + Prototyping with unicode names have passed the parser tests, further testing needs to be done. + +``` @Override + token Name = + ( UnicodeChar | '_' | '$' ) + ( UnicodeChar | '_' | '0'..'9' | '$' )*; + // Latin,Greek,Coptic,Cyrillic,Armenian + fragment token UnicodeChar = 'a'..'z' + |'A'..'Z' + |'\u00C0'..'\u00D6' + |'\u00D8'..'\u00F6' + |'\u00F8'..'\u02AF' + |'\u0370'..'\u0373' + |'\u0376' | '\u0377' | '\u037F' | '\u0386' + |'\u0386'..'\u03E1' + |'\u03E2'..'\u0481' + |'\u048A'..'\u0588'; +``` From 3cc3b667723d3d2eca7bc76a4eeb84d9086267d2 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 6 May 2026 17:38:58 +0200 Subject: [PATCH 21/26] Preperation completion --- Change documentation/Pullreguest.md | 40 +++++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/Change documentation/Pullreguest.md b/Change documentation/Pullreguest.md index 54a1951..ed8c42d 100644 --- a/Change documentation/Pullreguest.md +++ b/Change documentation/Pullreguest.md @@ -1,11 +1,12 @@ -# Pullrequest to incorporate new Python functionality. +# PullRequest to incorporate new Python functionality. In this Markdown all changes will be presented and reasoned. +To see what is known to be unsupported see knownToBeUnsupported.md ## Grammar ### Rework of For Decomposition Python currently allows another targets inside the for constructs, thereby we change our ForControl non terminal to be closer to the python language. -Old: +Old: ``` ForControl = ForDecomposition "in" ForIterable; interface ForDecomposition; @@ -37,7 +38,7 @@ target: identifier | "*" target ``` - Sources: https://docs.python.org/3/reference/simple_stmts.html#grammar-token-python-grammar-target_list and https://docs.python.org/3/reference/expressions.html (6.2.5) +Sources: https://docs.python.org/3/reference/simple_stmts.html#grammar-token-python-grammar-target_list and https://docs.python.org/3/reference/expressions.html (6.2.5) ### Rework of Boolean Instead of Defining a new BooleanLiteralPython, we override the Monticore BooleanLiteral to allow 'true' and 'false' as names, tests have shown that they can be used as such. @@ -67,12 +68,12 @@ With the following changes we aim combat these issues. ### Addition of complex numbers. Python supports the usage of complex numbers, they are formatted as: - (real number)? (+|-)? (imaginaryNumber)'j' +(real number)? (+|-) (imaginaryNumber)'j' A token PyComplexNumber was added supporting these numbers, additional a matching literal for their usage within python code. ``` token PyComplexNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; PyComplexNumberLiteral implements NumericLiteral <95> = PyComplexNumber; ``` -Source for cmath: https://docs.python.org/3/library/cmath.html +Source for cmath: https://docs.python.org/3/library/cmath.html (Experiments have shown that ### Addition of generics. We added support for generics as follows: @@ -83,9 +84,9 @@ We added support for generics as follows: ``` This is done similarly to https://docs.python.org/3/reference/compound_stmts.html#type-params. But differently to use the existing TypeAnnotation non terminal. To support the functionality described in the source above we added "GenericsAnnotation?" to: - - The the non terminals implementting the interface FunctionParameter. - - ClassFunctionDeclaration,ClassDeclaration,SimpleFunctionDeclaration,TypeDeclarationStatement - and the TypeAnnotations ``` GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation``` +- The the non terminals implementting the interface FunctionParameter. +- ClassFunctionDeclaration,ClassDeclaration,SimpleFunctionDeclaration,TypeDeclarationStatement + and the TypeAnnotations ``` GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation``` ### Statements @@ -127,12 +128,12 @@ Ours: ```symbol TypeRuleStatement implements Statement,ClassStatement = key("typ ##### Global So far we only allowed a single variable to be declared as global in a global statement. But Python allows multiple variables to be declared global in the same statement separated by commata see: https://docs.python.org/3/reference/simple_stmts.html#global , also so far we allowed a type annotation within a global statement which is not allowed, thereby this support is removed. -Additionally global can be used in classes as described in the source above. +Additionally global can be used in classes as described in the source above. Old: ```GlobalVariableDeclaration implements Statement = "global" Name (":" TypeAnnotation)? STATEMENT_END;``` New:```GlobalVariableDeclaration implements Statement,ClassStatement = "global" (Name || ",")+ STATEMENT_END;``` -##### Class Statement Block +##### Class Statement Block A ClassStatementBlock is now allowed to be a singular ClassStatement to allow classes such as: ``` class name1: pass @@ -162,13 +163,13 @@ CaseStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; Aditionally we allowed match statements inside classes. #### Try-Except -PEP758 Added support for leaving out parenthesis. Further so far the support for various Expressions and starred expressions were missing as specified in https://peps.python.org/pep-0758/ we added the support for this accordingly as specified. Also similarly to the new case statements, excepts now span a scope with the alias non terminal. +PEP758 Added support for leaving out parenthesis. Further so far the support for various Expressions and starred expressions were missing as specified in https://peps.python.org/pep-0758/ we added the support for this accordingly as specified. Also similarly to the new case statements, excepts now span a scope with the alias non terminal. Old: ``` ExceptStatement = "except" (PyQualifiedName? | "(" (PyQualifiedName || ",")+ ")") ("as" alias:Name)? ":" StatementBlock; ``` -New: +New: ``` scope ExceptStatement = "except" ExceptPattern? ":" ExceptStatementBlock; ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; @@ -217,12 +218,14 @@ New: ``` #### Comprehensions We allowed multiple if inside a comprehension by changing ? to * at the Generator filters. -This is allowed by python as specified here: https://docs.python.org/3/reference/expressions.html#grammar-token-python-grammar-comp_for +This is allowed by python as specified here: https://docs.python.org/3/reference/expressions.html#grammar-token-python-grammar-comp_for ### Another -We added some trailing ","? to rules where they were noticed to be allowed while testing. We also formated the grammar and added some comments +We added some trailing ","? to rules where they were noticed to be allowed while testing. We also formated the grammar and added some comments + ## Code In this section i will add the commits instead of the code snippets for readability reasons. + ### Preprocessor Commit: https://github.com/MontiCore/MCPython/commit/8c519eff8bcdf027ae421726fc36929f0bde2dd1 In line 75-77 i added that the continue line token is always ignored to allow statements like: @@ -231,11 +234,14 @@ In line 75-77 i added that the continue line token is always ignored to allow st var3, var4) ``` as these could cause issues. + ### Visitor and CoCos related to aliased expressions. -To avoid aliased expressions to be used anywhere we added a matching CoCo and Visitor. +Marc suggested to add a Coco and Visitor to implement the prevention of aliased expressions to be used outside the match blocks, i implemented this in the following commit. Commit: https://github.com/MontiCore/MCPython/commit/ba9e8f18fb1c038a2cca33527a1e1bb8552778ab ### Adaptation of new tests We removed a test regarding invalid classes that are now valid (because for loops are now allowed as class statements). -We added new test cases for Python -...to do more specific... +We added new test cases for Python, for that we looked what the python parser tests and we do not test. +We then added some tests. + +See: https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py , and the commit https://github.com/MontiCore/MCPython/commit/78659912ca6455b515f1514dcaefbc887ffa0133 \ No newline at end of file From 14a1c17f52ffa4bf0378005aafa95b5cff717bb9 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 6 May 2026 17:41:11 +0200 Subject: [PATCH 22/26] small fix --- Change documentation/Pullreguest.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Change documentation/Pullreguest.md b/Change documentation/Pullreguest.md index ed8c42d..8aa16fc 100644 --- a/Change documentation/Pullreguest.md +++ b/Change documentation/Pullreguest.md @@ -68,12 +68,12 @@ With the following changes we aim combat these issues. ### Addition of complex numbers. Python supports the usage of complex numbers, they are formatted as: -(real number)? (+|-) (imaginaryNumber)'j' +(real number) (+|-) (imaginaryNumber)'j' A token PyComplexNumber was added supporting these numbers, additional a matching literal for their usage within python code. ``` token PyComplexNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; PyComplexNumberLiteral implements NumericLiteral <95> = PyComplexNumber; ``` -Source for cmath: https://docs.python.org/3/library/cmath.html (Experiments have shown that +Source for cmath: https://docs.python.org/3/library/cmath.html (Experiments have shown that that the real number and sign of the imaginary can be left out) ### Addition of generics. We added support for generics as follows: @@ -244,4 +244,4 @@ We removed a test regarding invalid classes that are now valid (because for loop We added new test cases for Python, for that we looked what the python parser tests and we do not test. We then added some tests. -See: https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py , and the commit https://github.com/MontiCore/MCPython/commit/78659912ca6455b515f1514dcaefbc887ffa0133 \ No newline at end of file +See: https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py , and the commit https://github.com/MontiCore/MCPython/commit/78659912ca6455b515f1514dcaefbc887ffa0133 From 2cd74fe2c1ed99e93da32ff456685e4fd44cb839 Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 6 May 2026 17:44:36 +0200 Subject: [PATCH 23/26] fix minor wrong indentation --- .../_parser/WhitespaceSensitiveProcessor.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/de/monticore/python/_parser/WhitespaceSensitiveProcessor.java b/src/main/java/de/monticore/python/_parser/WhitespaceSensitiveProcessor.java index 35b2439..80d5d49 100644 --- a/src/main/java/de/monticore/python/_parser/WhitespaceSensitiveProcessor.java +++ b/src/main/java/de/monticore/python/_parser/WhitespaceSensitiveProcessor.java @@ -25,14 +25,14 @@ public class WhitespaceSensitiveProcessor { private final TokenFactory tokenFactory; public WhitespaceSensitiveProcessor( - Pair source, - List indents, - List emptyLines, - TokenFactory tokenFactory, - Token eolTokenProto, - Token incIndentTokenProto, - Token decIndentTokenProto, - PreprocessingTokens preprocessingTokens + Pair source, + List indents, + List emptyLines, + TokenFactory tokenFactory, + Token eolTokenProto, + Token incIndentTokenProto, + Token decIndentTokenProto, + PreprocessingTokens preprocessingTokens ) { this.source = source; this.indents = indents; From 6aa44a289bf4532612e15710b269da289cdc13cd Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 6 May 2026 17:48:25 +0200 Subject: [PATCH 24/26] fix minor wrong indentation --- ...sedWhitespacePreprocessingTokenSource.java | 30 +++++++++---------- .../_parser/WhitespaceSensitiveProcessor.java | 16 +++++----- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java b/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java index 4879db3..bdd2a32 100644 --- a/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java +++ b/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java @@ -26,27 +26,27 @@ public class StateBasedWhitespacePreprocessingTokenSource { private PreprocessingTokens preprocessingTokens; public StateBasedWhitespacePreprocessingTokenSource( - Pair source, - List indents, - List emptyLines, - Lexer delegate, - Token eolTokenProto, - Token incIndentTokenProto, - Token decIndentTokenProto, - PreprocessingTokens preprocessingTokens + Pair source, + List indents, + List emptyLines, + Lexer delegate, + Token eolTokenProto, + Token incIndentTokenProto, + Token decIndentTokenProto, + PreprocessingTokens preprocessingTokens ) { this.openingParens = List.of(preprocessingTokens.lparenTokenType, - preprocessingTokens.lCurlyParenTokenType, - preprocessingTokens.lSquareParenTokenType); + preprocessingTokens.lCurlyParenTokenType, + preprocessingTokens.lSquareParenTokenType); this.closingParens = List.of(preprocessingTokens.rparenTokenType, - preprocessingTokens.rCurlyParenTokenType, - preprocessingTokens.rSquareParenTokenType); + preprocessingTokens.rCurlyParenTokenType, + preprocessingTokens.rSquareParenTokenType); this.sensitiveProcessor = new WhitespaceSensitiveProcessor( - source, indents, emptyLines, delegate.getTokenFactory(), - eolTokenProto, incIndentTokenProto, decIndentTokenProto, - preprocessingTokens + source, indents, emptyLines, delegate.getTokenFactory(), + eolTokenProto, incIndentTokenProto, decIndentTokenProto, + preprocessingTokens ); this.lastToken = null; diff --git a/src/main/java/de/monticore/python/_parser/WhitespaceSensitiveProcessor.java b/src/main/java/de/monticore/python/_parser/WhitespaceSensitiveProcessor.java index 80d5d49..35b2439 100644 --- a/src/main/java/de/monticore/python/_parser/WhitespaceSensitiveProcessor.java +++ b/src/main/java/de/monticore/python/_parser/WhitespaceSensitiveProcessor.java @@ -25,14 +25,14 @@ public class WhitespaceSensitiveProcessor { private final TokenFactory tokenFactory; public WhitespaceSensitiveProcessor( - Pair source, - List indents, - List emptyLines, - TokenFactory tokenFactory, - Token eolTokenProto, - Token incIndentTokenProto, - Token decIndentTokenProto, - PreprocessingTokens preprocessingTokens + Pair source, + List indents, + List emptyLines, + TokenFactory tokenFactory, + Token eolTokenProto, + Token incIndentTokenProto, + Token decIndentTokenProto, + PreprocessingTokens preprocessingTokens ) { this.source = source; this.indents = indents; From 32eafdefdf38b267a7474e76502ce2c0539ec1ec Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Wed, 6 May 2026 17:51:11 +0200 Subject: [PATCH 25/26] fix minor wrong indentation --- ...sedWhitespacePreprocessingTokenSource.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java b/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java index bdd2a32..9169e66 100644 --- a/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java +++ b/src/main/java/de/monticore/python/_parser/StateBasedWhitespacePreprocessingTokenSource.java @@ -26,27 +26,27 @@ public class StateBasedWhitespacePreprocessingTokenSource { private PreprocessingTokens preprocessingTokens; public StateBasedWhitespacePreprocessingTokenSource( - Pair source, - List indents, - List emptyLines, - Lexer delegate, - Token eolTokenProto, - Token incIndentTokenProto, - Token decIndentTokenProto, - PreprocessingTokens preprocessingTokens + Pair source, + List indents, + List emptyLines, + Lexer delegate, + Token eolTokenProto, + Token incIndentTokenProto, + Token decIndentTokenProto, + PreprocessingTokens preprocessingTokens ) { this.openingParens = List.of(preprocessingTokens.lparenTokenType, - preprocessingTokens.lCurlyParenTokenType, - preprocessingTokens.lSquareParenTokenType); + preprocessingTokens.lCurlyParenTokenType, + preprocessingTokens.lSquareParenTokenType); this.closingParens = List.of(preprocessingTokens.rparenTokenType, - preprocessingTokens.rCurlyParenTokenType, - preprocessingTokens.rSquareParenTokenType); + preprocessingTokens.rCurlyParenTokenType, + preprocessingTokens.rSquareParenTokenType); this.sensitiveProcessor = new WhitespaceSensitiveProcessor( - source, indents, emptyLines, delegate.getTokenFactory(), - eolTokenProto, incIndentTokenProto, decIndentTokenProto, - preprocessingTokens + source, indents, emptyLines, delegate.getTokenFactory(), + eolTokenProto, incIndentTokenProto, decIndentTokenProto, + preprocessingTokens ); this.lastToken = null; From 1f6f22132557c051e7bcff68e7618b23df50ab0d Mon Sep 17 00:00:00 2001 From: Maurice Rosenbaum Date: Sun, 17 May 2026 14:16:53 +0200 Subject: [PATCH 26/26] Fixing Case Statements to allow the capturing of subpatterns decribed in PEP 636 and PEP 642 and https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-patterns Fixing Wrong Formatting Adding Char related comment Deleting obsolete Visitor Deleting Pullreguest.md Restructuring move knownToBeUnsupported.md to root directory --- Change documentation/Pullreguest.md | 247 ------- ...eUnsupported.md => knownToBeUnsupported.md | 0 src/main/grammars/de/monticore/Python.mc4 | 695 ++++++++++-------- .../python/_cocos/CaseStatementVisitor.java | 36 - .../_cocos/ExpressionsCorrectlyAliased.java | 19 - 5 files changed, 370 insertions(+), 627 deletions(-) delete mode 100644 Change documentation/Pullreguest.md rename Change documentation/knownToBeUnsupported.md => knownToBeUnsupported.md (100%) delete mode 100644 src/main/java/de/monticore/python/_cocos/CaseStatementVisitor.java delete mode 100644 src/main/java/de/monticore/python/_cocos/ExpressionsCorrectlyAliased.java diff --git a/Change documentation/Pullreguest.md b/Change documentation/Pullreguest.md deleted file mode 100644 index 8aa16fc..0000000 --- a/Change documentation/Pullreguest.md +++ /dev/null @@ -1,247 +0,0 @@ -# PullRequest to incorporate new Python functionality. -In this Markdown all changes will be presented and reasoned. -To see what is known to be unsupported see knownToBeUnsupported.md -## Grammar - -### Rework of For Decomposition -Python currently allows another targets inside the for constructs, thereby we change our ForControl non terminal to be closer to the python language. - -Old: -``` - ForControl = ForDecomposition "in" ForIterable; - interface ForDecomposition; - ForDecompositionComma implements ForDecomposition = ForDecomposition "," (ForDecomposition ","?)?; - ForDecompositionParenthesis implements ForDecomposition = "("ForDecomposition ")"; - ForDecompositionBrackets implements ForDecomposition = "[" ForDecomposition "]"; - ForStarredVariable implements ForDecomposition = "*" ForDecomposition; -``` -New: -``` - ForControl = ForList "in" ForIterable; - interface ForDecomposition; - ForList = (ForDecomposition || ",")+ ","?; - ForDecompositionParenthesis implements ForDecomposition = "(" ForList? ")"; - ForDecompositionBrackets implements ForDecomposition = "[" ForList? "]"; - ForStarredVariable implements ForDecomposition = "*" ForDecomposition ; - ForPyQualifiedName implements ForDecomposition = PyQualifiedName; -``` -Python: ("[...]" Marks optional) -``` -comprehension: assignment_expression comp_for -comp_for: ["async"] "for" target_list "in" or_test [comp_iter] -target_list: target ("," target)* [","] -target: identifier - | "(" [target_list] ")" - | "[" [target_list] "]" - | attributeref - | subscription - | "*" target - -``` -Sources: https://docs.python.org/3/reference/simple_stmts.html#grammar-token-python-grammar-target_list and https://docs.python.org/3/reference/expressions.html (6.2.5) - -### Rework of Boolean -Instead of Defining a new BooleanLiteralPython, we override the Monticore BooleanLiteral to allow 'true' and 'false' as names, tests have shown that they can be used as such. - -### Rework of fstrings -It proved to be difficult to support all variants of fStrings, so far they were parsed as a singular string token, thereby the content of curly brackets was considered text, yet tests have shown that line breaks are allowed within curly brackets. Another issue is the allowed usage of the same quotes " or ' in the outer string as well as inside the curly brackets. Finally it is possible to add f to chars in python such as ```f'a'``` this was so far an issue as only strings were allowed to be modified and not chars. -With the following changes we aim combat these issues. - -1) We don't allow f and F as string modifier (line 121) -2) We add chars to the string definitions (line 115,line 120) -3) We add new tokens for fstrings - ``` - //Double and single quoted strings with an f modifier f'text1{exp}text2', separate definition to allow more expressions. - token FSQStringPython - = ('f'|'F') '\'' (StringFSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; - token FDQStringPython - = ('f'|'F') '"' (StringFDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; - fragment token StringFSQCharactersPython = (StringFSQCharacterPython)+; - fragment token StringFDQCharactersPython = (StringFDQCharacterPython)+; - fragment token StringFSQCharacterPython = ~ ('\''| '\\'| '{' )| PythonEscapeSequence |InnerFString; - fragment token StringFDQCharacterPython = ~ ('"'| '\\'| '{') | PythonEscapeSequence |InnerFString; - fragment token InnerFString = '{' ~('}')* '}'; - ``` -4) We similarly define fstring literals and non terminals for multilinestring tokens -5) We add a new FStringPython non terminal (line 120) and add it to the StringLiteralPython (line 112-117) - - -### Addition of complex numbers. -Python supports the usage of complex numbers, they are formatted as: -(real number) (+|-) (imaginaryNumber)'j' -A token PyComplexNumber was added supporting these numbers, additional a matching literal for their usage within python code. -``` token PyComplexNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; - PyComplexNumberLiteral implements NumericLiteral <95> = PyComplexNumber; -``` -Source for cmath: https://docs.python.org/3/library/cmath.html (Experiments have shown that that the real number and sign of the imaginary can be left out) -### Addition of generics. - -We added support for generics as follows: -``` - GenericsAnnotation = "[" Generics? "]"; - Generics = (Generic || ",")+ ; - Generic = TypeAnnotation (":" TypeAnnotation)?; -``` -This is done similarly to https://docs.python.org/3/reference/compound_stmts.html#type-params. -But differently to use the existing TypeAnnotation non terminal. To support the functionality described in the source above we added "GenericsAnnotation?" to: -- The the non terminals implementting the interface FunctionParameter. -- ClassFunctionDeclaration,ClassDeclaration,SimpleFunctionDeclaration,TypeDeclarationStatement - and the TypeAnnotations ``` GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation``` - -### Statements - -### ClassStatements -Import, If, Assert, For, While, ConditionalExecution, With, GlobalVariableDeclaration, NonLocalVariableDeclaration, MultiVariableDeclaration, ParenMultiVariableDeclaration, Raise, TypeRuleStatement,Delete statements are now implementing the ClassStatement interface. By experimenting and testing they showed to be allowed inside classes. - -#### New Statements - -##### Nonlocal -For a detailed description of the nonlocal keyword see: https://docs.python.org/3/tutorial/classes.html under 9.2 . Summarizing the source, nonlocal allows a function or class defined inside another function to accesses the enclosing functions variables. Without the non local keyword these variables are read only to nested functions and classes. Our implementation is similar to pythons https://docs.python.org/3/reference/simple_stmts.html#nonlocal. - -Pythons: ```nonlocal_stmt: "nonlocal" identifier ("," identifier)* ```(see source above) -Ours : ```NonLocalVariableDeclaration implements Statement,ClassStatement = "nonlocal" names:(Name || ",")+ STATEMENT_END;``` - -##### Yield from -Yield from statements in the form: 'yield' 'from' expression have been added in https://peps.python.org/pep-0380/ ,we support them similarly to python https://docs.python.org/3/reference/expressions.html#grammar-token-python-grammar-yield_expression. - -Pythons: ```yield_from: "yield" "from" expression``` (see source above) -Ours:```YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; ``` - - -##### Type Declaration Statement -To allow type hints as ``` var : int``` without an assignment (in comparison to augmented assignments), we add the new TypeDeclartationStatement. -``` -TypeDeclaration implements Statement = Expression ":" TypeAnnotation STATEMENT_END; -``` -Python incorporates this functionality in the augmented assignments, yet this caused issues for us. see https://docs.python.org/3/reference/simple_stmts.html#index-15. - -##### Type Aliases -Python allows to declare type aliases. Where a type is aliased under an identifier, see https://docs.python.org/3/reference/simple_stmts.html#grammar-token-python-grammar-type_stmt. - -We implement the support similarly, also we define the type alias as symbol. -Python: ```type_stmt: 'type' identifier [type_params] "=" expression``` (see source above) -Ours: ```symbol TypeRuleStatement implements Statement,ClassStatement = key("type") Name GenericsAnnotation? "=" Expression STATEMENT_END;``` -'type' needs to be declared as local keyword as tests have shown that it can be used as variable name. Python also specifies this https://docs.python.org/3/reference/lexical_analysis.html#soft-keywords . - -#### Changes - -##### Global -So far we only allowed a single variable to be declared as global in a global statement. -But Python allows multiple variables to be declared global in the same statement separated by commata see: https://docs.python.org/3/reference/simple_stmts.html#global , also so far we allowed a type annotation within a global statement which is not allowed, thereby this support is removed. -Additionally global can be used in classes as described in the source above. - -Old: ```GlobalVariableDeclaration implements Statement = "global" Name (":" TypeAnnotation)? STATEMENT_END;``` -New:```GlobalVariableDeclaration implements Statement,ClassStatement = "global" (Name || ",")+ STATEMENT_END;``` - -##### Class Statement Block -A ClassStatementBlock is now allowed to be a singular ClassStatement to allow classes such as: -``` -class name1: pass -class name2: a=1 -``` -Old: ```ClassStatementBlock = BLOCK_START ClassStatementBlockBody BLOCK_END;``` -New: ```ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement;``` - -Test have shown that this is needed. - -#### Match Statement -Case statements in python allow patterns to be aliased (as example ```case Type as t:```) see https://docs.python.org/3/reference/compound_stmts.html#the-match-statement and https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-patterns. To allow this, case statements now span a scope to capture the alias. -Old: -``` -MatchStatement implements Statement = key("match") Expression ":" MatchBlock; -scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; -CaseStatement = key("case") (Expression || "|")+ ("if" condition:Expression)? ":" StatementBlock; -``` -New: -``` -MatchStatement implements Statement,ClassStatement = key("match") Expression ":" MatchBlock; -scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; -scope CaseStatement = key("case") (Expression || "|")+ ("as" Alias)? ("if" condition:Expression)? ":" CaseStatementBlock; -CaseStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; -``` -(Alias is defined as ```symbol Alias = Name;````) -Aditionally we allowed match statements inside classes. - -#### Try-Except -PEP758 Added support for leaving out parenthesis. Further so far the support for various Expressions and starred expressions were missing as specified in https://peps.python.org/pep-0758/ we added the support for this accordingly as specified. Also similarly to the new case statements, excepts now span a scope with the alias non terminal. -Old: -``` - ExceptStatement = "except" (PyQualifiedName? | "(" (PyQualifiedName || ",")+ ")") ("as" alias:Name)? ":" StatementBlock; - -``` -New: -``` -scope ExceptStatement = "except" ExceptPattern? ":" ExceptStatementBlock; -ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; -//PEP758 -interface ExceptPattern; -ExpressionListing implements ExceptPattern = (Expression || ",")+; -ParenthesisedExpressionListing implements ExceptPattern = "(" (Expression || ",")+ ")" ("as" Alias)?; -StarredExpressionListing implements ExceptPattern = "*"(Expression || ",")+; -StarredParenthesisedExpressionListing implements ExceptPattern = "*""(" (Expression || ",")+ ")" ("as" Alias)?; -``` -### Expressions - -#### Changes - -##### Assignment -As defined by https://docs.python.org/3/reference/simple_stmts.html#index-14 python's assignment statements additionally supports "//=" "@=" "**=" thereby we add this support by overriding the AssignmentExpression from Monticore by simply adding the three additional operators to the square brackets. -``` - @Override - AssignmentExpression implements Expression <60> = - left:Expression - operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" - | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] - right:Expression; - -``` -##### Augmented Assignment -So far we allowed the same operators as in the assignment expression, yet python only allows "=" see https://docs.python.org/3/reference/simple_stmts.html#index-15. - -Thereby we remove the square brackets and only allow "=". -Old: -``` - AnnotatedAssignmentExpression implements Expression <60> = - left:Expression - ":" annotated: TypeAnnotation - operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" - | "^=" | ">>=" | ">>>=" | "<<=" | "%=" ] - right:Expression; -``` -New: -``` - AnnotatedAssignmentExpression implements Expression <60> = - left:Expression - ":" annotated: TypeAnnotation - "=" - right:Expression; -``` -#### Comprehensions -We allowed multiple if inside a comprehension by changing ? to * at the Generator filters. -This is allowed by python as specified here: https://docs.python.org/3/reference/expressions.html#grammar-token-python-grammar-comp_for - -### Another -We added some trailing ","? to rules where they were noticed to be allowed while testing. We also formated the grammar and added some comments - -## Code -In this section i will add the commits instead of the code snippets for readability reasons. - -### Preprocessor -Commit: https://github.com/MontiCore/MCPython/commit/8c519eff8bcdf027ae421726fc36929f0bde2dd1 -In line 75-77 i added that the continue line token is always ignored to allow statements like: -``` - function(var1, var2,\ - var3, var4) -``` -as these could cause issues. - -### Visitor and CoCos related to aliased expressions. -Marc suggested to add a Coco and Visitor to implement the prevention of aliased expressions to be used outside the match blocks, i implemented this in the following commit. -Commit: https://github.com/MontiCore/MCPython/commit/ba9e8f18fb1c038a2cca33527a1e1bb8552778ab - -### Adaptation of new tests -We removed a test regarding invalid classes that are now valid (because for loops are now allowed as class statements). -We added new test cases for Python, for that we looked what the python parser tests and we do not test. -We then added some tests. - -See: https://github.com/python/cpython/blob/3.9/Lib/test/test_parser.py , and the commit https://github.com/MontiCore/MCPython/commit/78659912ca6455b515f1514dcaefbc887ffa0133 diff --git a/Change documentation/knownToBeUnsupported.md b/knownToBeUnsupported.md similarity index 100% rename from Change documentation/knownToBeUnsupported.md rename to knownToBeUnsupported.md diff --git a/src/main/grammars/de/monticore/Python.mc4 b/src/main/grammars/de/monticore/Python.mc4 index 1c652d9..43cc6f7 100644 --- a/src/main/grammars/de/monticore/Python.mc4 +++ b/src/main/grammars/de/monticore/Python.mc4 @@ -24,361 +24,406 @@ grammar Python extends MultilineString, // Must be first to avoid conflicts wit PythonScript = Statement*; /*====================================== Tokens ======================================*/ - @Override - token WS = (' ' | '\t' | '\r' | '\n' ) : -> channel(HIDDEN); - @Override - token SL_COMMENT = "#" (~('\n' | '\r' ))* : -> channel(HIDDEN); - token ByteOrderMark = '\uFEFF' : -> skip; - - /** - * The following utf-8 symbols are used to parse the Python files without having to add whitespace to the Grammar. - * In a preprocessing step code blocks are denoted with \u204f = ⦃ and \u2984 = ⦄, and lines are ended with \u204f = ⁏ - */ - token BLOCK_START = '\u2983'; - token BLOCK_END = '\u2984'; - token STATEMENT_END = ';'? '\u204f' | ';' '\u204f'?; - - // Will be filtered out by the WhitespacePreprocessingTokenSource - // Used break a line without finishing the statement - token CONTINUE_LINE_TOKEN = '\\' '\r'? '\n'; - - //String tokens for python - - //Often (mis)used as a multiline comment but can also be used as a string literal, thus we can not skip it - token MultiLineStringToken = ((("\'\'\'") .*? ("\'\'\'")) | ((DoubleQuoteMultilineStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); - token MultiLineFStringToken = ((("\'\'\'") .*? ("\'\'\'")) |((DoubleQuoteMultilineFStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); - - //Double quoted Strings "Text" - @Override - token String = '"' (StringDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; - fragment token StringDQCharactersPython = (StringDQCharacterPython)+; - fragment token StringDQCharacterPython = ~ ('"' | '\\' | '\n') | PythonEscapeSequence; - - //Single quoted Strings 'Text' - token StringPython = '\'' (StringSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; - fragment token StringSQCharactersPython = (StringSQCharacterPython)+; - fragment token StringSQCharacterPython = ~ ('\''| '\\' | '\n') | PythonEscapeSequence; - - //Escape in strings "\n" - fragment token PythonEscapeSequence = '\\' .; - - //Double and single quoted strings with an f modifier f'text1{exp}text2', separate definition to allow more expressions. - token FSQStringPython - = ('f'|'F') '\'' (StringFSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; - token FDQStringPython - = ('f'|'F') '"' (StringFDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; - fragment token StringFSQCharactersPython = (StringFSQCharacterPython)+; - fragment token StringFDQCharactersPython = (StringFDQCharacterPython)+; - fragment token StringFSQCharacterPython = ~ ('\''| '\\'| '{' )| PythonEscapeSequence |InnerFString; - fragment token StringFDQCharacterPython = ~ ('"'| '\\'| '{') | PythonEscapeSequence |InnerFString; - fragment token InnerFString = '{' ~('}')* '}'; - - //number tokens for python - - token FloatWithExponent = (DigitsPart | PyFloat) ('e'|'E') ('+' | '-')? DigitsPart; - FloatWithExponentLiteral implements NumericLiteral <100> = FloatWithExponent; - - token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; - PyFloatLiteral implements NumericLiteral <200> = PyFloat; - - token PyComplexNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; - PyComplexNumberLiteral implements NumericLiteral <95> = PyComplexNumber; - - //PEP 515 - token DigitsPart = Digit ('_'? Digit)*; - - token HexNumberToken = '0' 'x' ('0'..'9' | 'a'..'f' | 'A'..'F')+; - - //PEP 515: Underscores in Numeric Literals - @Override - token Digits = Digit ('_'? Digit)*; // technically the first digit must be nonzero except for 0(_0)* + @Override + token WS = (' ' | '\t' | '\r' | '\n' ) : -> channel(HIDDEN); + @Override + token SL_COMMENT = "#" (~('\n' | '\r' ))* : -> channel(HIDDEN); + token ByteOrderMark = '\uFEFF' : -> skip; + + /** + * The following utf-8 symbols are used to parse the Python files without having to add whitespace to the Grammar. + * In a preprocessing step code blocks are denoted with \u204f = ⦃ and \u2984 = ⦄, and lines are ended with \u204f = ⁏ + */ + token BLOCK_START = '\u2983'; + token BLOCK_END = '\u2984'; + token STATEMENT_END = ';'? '\u204f' | ';' '\u204f'?; + + // Will be filtered out by the WhitespacePreprocessingTokenSource + // Used break a line without finishing the statement + token CONTINUE_LINE_TOKEN = '\\' '\r'? '\n'; + + // === string tokens for python === + //Often (mis)used as a multiline comment but can also be used as a string literal, thus we can not skip it + token MultiLineStringToken = ((("\'\'\'") .*? ("\'\'\'")) | ((DoubleQuoteMultilineStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); + token MultiLineFStringToken = ((("\'\'\'") .*? ("\'\'\'")) |((DoubleQuoteMultilineFStringDelimiter .*? DoubleQuoteMultilineStringDelimiter))); + + //Double quoted Strings "Text" + @Override + token String = '"' (StringDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; + + fragment token StringDQCharactersPython + = (StringDQCharacterPython)+; + fragment token StringDQCharacterPython + = ~ ('"'| '\\' | '\n') | PythonEscapeSequence; + + //Single quoted Strings 'Text' + token StringPython = '\'' (StringSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; + + fragment token StringSQCharactersPython + = (StringSQCharacterPython)+; + fragment token StringSQCharacterPython + = ~ ('\''| '\\' | '\n') | PythonEscapeSequence; + + //Escape in strings "\n" + fragment token PythonEscapeSequence = '\\' .; + + //Double and single quoted strings with an f modifier f'text1{exp}text2', separate definition to allow more expressions. + token FSQStringPython + = ('f'|'F') '\'' (StringFSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));}; + token FDQStringPython + = ('f'|'F') '"' (StringFDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));}; + + fragment token StringFSQCharactersPython + = (StringFSQCharacterPython)+; + fragment token StringFDQCharactersPython + = (StringFDQCharacterPython)+; + + fragment token StringFSQCharacterPython + = ~ ('\''| '\\'| '{' )| PythonEscapeSequence |InnerFString; + fragment token StringFDQCharacterPython + = ~ ('"'| '\\'| '{') | PythonEscapeSequence |InnerFString; + fragment token InnerFString = '{' ~('}')* '}'; + + // === number tokens for python === + + token FloatWithExponent = (DigitsPart | PyFloat) ('e'|'E') ('+' | '-')? DigitsPart; + FloatWithExponentLiteral implements NumericLiteral <100> = FloatWithExponent; + + token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.'; + PyFloatLiteral implements NumericLiteral <200> = PyFloat; + + token PyComplexNumber = (DigitsPart | PyFloat)? ('+' | '-')? DigitsPart "j"; + PyComplexNumberLiteral implements NumericLiteral <95> = PyComplexNumber; + + // PEP 515 + token DigitsPart = Digit ('_'? Digit)*; + + token HexNumberToken = '0' 'x' ('0'..'9' | 'a'..'f' | 'A'..'F')+; + // PEP 515: Underscores in Numeric Literals + @Override + token Digits + = Digit ('_'? Digit)*; // technically the first digit must be nonzero except for 0(_0)* /*====================================== Literals ======================================*/ - HexNumberLiteral implements NumericLiteral <100> = HexNumberToken; + HexNumberLiteral implements NumericLiteral <100> = HexNumberToken; - //Literals for Datastructures + //Literals for Datastructures + ArrayLiteral implements Literal = "[" (VariableInit || ",")* ","? "]" ; + TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ; + DictLiteral implements Literal = "{" (DictEntry || ",")* ","? "}" ; + SetLiteral implements Literal = "{" (Expression || ",")* ","? "}" ; - ArrayLiteral implements Literal = "[" (VariableInit || ",")* ","? "]" ; - TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ; - DictLiteral implements Literal = "{" (DictEntry || ",")* ","? "}" ; - SetLiteral implements Literal = "{" (Expression || ",")* ","? "}" ; - DictEntry = key:VariableInit ":" value:VariableInit | SpreadMappingExpression; + DictEntry = key:VariableInit ":" value:VariableInit | SpreadMappingExpression; - //Literals and helper-definitions regarding Strings + //Literals and helper-definitions regarding Strings + MultiLineStringLiteral implements Literal = (StringModifier)? MultiLineStringToken; + MultiLineFStringLiteral implements Literal = (StringModifier)? MultiLineFStringToken; - MultiLineStringLiteral implements Literal = (StringModifier)? MultiLineStringToken; - MultiLineFStringLiteral implements Literal = (StringModifier)? MultiLineFStringToken; - StringLiteralPython implements Literal, SignedLiteral = - ( - StringModifier? - (source:StringPython | source:String |source:Char) - ) - | fsource: FStringPython; + //Char is necessary because single quoted characters will be recognized as char tokens not string tokens, + // as char is imported from MCCommonLiterals.mc4 + StringLiteralPython implements Literal, SignedLiteral = + ( + StringModifier? + (source:StringPython | source:String |source:Char) + ) + | fsource: FStringPython; - StringsLiteralPython implements Literal <200> = (StringLiteralPython | StringLiteral | MultiLineStringLiteral | MultiLineFStringLiteral | FStringPython)+; - FStringPython = (FSQStringPython | FDQStringPython | ("f"|"F") Char); - StringModifier = /*{cmpTokenRegEx(1, "(r|b|u|R|B|U)+")}?*/ type:Name; + StringsLiteralPython implements Literal <200> = (StringLiteralPython | StringLiteral | MultiLineStringLiteral | MultiLineFStringLiteral | FStringPython)+; - //boolean literals for python - @Override - BooleanLiteral implements Literal, SignedLiteral = ource:["True" | "False"]; + FStringPython = (FSQStringPython | FDQStringPython | ("f"|"F") Char); + StringModifier = /*{cmpTokenRegEx(1, "(r|b|u|R|B|U)+")}?*/ type:Name; - //https://docs.python.org/dev/library/constants.html#Ellipsis - EllipsisLiteral implements Literal = "..."; - splittoken "..."; + // boolean literals for python + @Override + BooleanLiteral implements Literal, SignedLiteral = + source:["True" | "False"]; + + // https://docs.python.org/dev/library/constants.html#Ellipsis + EllipsisLiteral implements Literal = "..."; + splittoken "..."; /*====================================== Statements ======================================*/ interface Statement; - //The StatementBlock represents a number of statements with the same indentation - - scope StatementBlock = (BLOCK_START StatementBlockBody BLOCK_END) | Statement; - StatementBlockBody = Statement+; - - //Various statements - - //Helper-definition for qualified names in python. - PyQualifiedName = (Name || ".")+; - astrule PyQualifiedName = method public String joined(){ - return String.join(".", getNameList()); - }; - - //import statement - ImportStatement implements Statement ,ClassStatement = - ("from" {!next("import")}? (leadingDots:"."*) (module:PyQualifiedName)?)? - "import" - ( - ModuleList - | paren:"(" ModuleList ")" - ) - STATEMENT_END; - ModuleList = star:"*" | ((ModuleWithOptionalAlias || ",")+ ","?); - ModuleWithOptionalAlias = name:PyQualifiedName ("as" alias:Name)?; - - //Conditional statements - IfStatement implements Statement,ClassStatement = "if" condition:Expression ":" thenStatement:StatementBlock - ("elif" elifCondition:Expression ":" elifStatement:StatementBlock )* - ElseStatementPart?; - ElseStatementPart = "else" ":" StatementBlock; - ConditionalExecutionStatement implements Statement ,ClassStatement= "if" condition: Expression ":" Expression; - - //Assert statement - AssertStatement implements Statement,ClassStatement = "assert" condition:Expression ("," errorMessage:Expression)? STATEMENT_END; - - //for statement - scope ForStatement implements Statement,ClassStatement = async:"async"? "for" ForControl ":" StatementBlock - ElseStatementPart? ; - ForControl = ForList "in" ForIterable; - interface ForDecomposition; - ForList = (ForDecomposition || ",")+ ","?; - ForVariable implements Variable, ForDecomposition = Name; - ForDecompositionParenthesis implements ForDecomposition = "(" ForList? ")"; - ForDecompositionBrackets implements ForDecomposition = "[" ForList? "]"; - ForStarredVariable implements ForDecomposition = "*" ForDecomposition ; - ForPyQualifiedName implements ForDecomposition = PyQualifiedName; - ForIterable = Expression; - - //while statement - WhileStatement implements Statement,ClassStatement = "while" condition:Expression ":" StatementBlock - ElseStatementPart? ; - - //Control-flow statements - PassStatement implements Statement, ClassStatement = "pass" STATEMENT_END; - BreakStatement implements Statement = "break" STATEMENT_END; - ContinueStatement implements Statement = "continue" STATEMENT_END; - - //try-except-finally statement - TryExceptStatement implements Statement,ClassStatement = "try" ":" tryStatement:StatementBlock + // The StatementBlock represents a number of statements with the same indentation + scope StatementBlock = (BLOCK_START StatementBlockBody BLOCK_END) | Statement; + StatementBlockBody = Statement+; + + //Helper-definition for qualified names in python. + PyQualifiedName = (Name || ".")+; + astrule PyQualifiedName = method public String joined(){ + return String.join(".", getNameList()); + }; + + // import statement + ImportStatement implements Statement,ClassStatement = + ("from" {!next("import")}? (leadingDots:"."*) (module:PyQualifiedName)?)? + "import" + ( + ModuleList + | paren:"(" ModuleList ")" + ) + STATEMENT_END; + + ModuleList = star:"*" | ((ModuleWithOptionalAlias || ",")+ ","?); + + ModuleWithOptionalAlias = name:PyQualifiedName ("as" alias:Name)?; + + //Conditional statements + IfStatement implements Statement,ClassStatement = "if" condition:Expression ":" thenStatement:StatementBlock + ("elif" elifCondition:Expression ":" elifStatement:StatementBlock )* + ElseStatementPart?; + ElseStatementPart = "else" ":" StatementBlock; + ConditionalExecutionStatement implements Statement ,ClassStatement= "if" condition: Expression ":" Expression; + + //assert statement + AssertStatement implements Statement,ClassStatement = "assert" condition:Expression ("," errorMessage:Expression)? STATEMENT_END; + + //for statement + scope ForStatement implements Statement,ClassStatement = async:"async"? "for" ForControl ":" StatementBlock + ElseStatementPart? ; + + ForControl = ForList "in" ForIterable; + interface ForDecomposition; + ForList = (ForDecomposition || ",")+ ","?; + ForVariable implements Variable, ForDecomposition = Name; + ForDecompositionParenthesis implements ForDecomposition = "(" ForList? ")"; + ForDecompositionBrackets implements ForDecomposition = "[" ForList? "]"; + ForStarredVariable implements ForDecomposition = "*" ForDecomposition ; + ForPyQualifiedName implements ForDecomposition = PyQualifiedName; + ForIterable = Expression; + + //while statement + WhileStatement implements Statement,ClassStatement = "while" condition:Expression ":" StatementBlock + ElseStatementPart? ; + + //Control-flow statements + PassStatement implements Statement, ClassStatement = "pass" STATEMENT_END; + BreakStatement implements Statement = "break" STATEMENT_END; + ContinueStatement implements Statement = "continue" STATEMENT_END; + + // try-except-finally statement + TryExceptStatement implements Statement,ClassStatement = "try" ":" tryStatement:StatementBlock + ( ( - ( - ExceptStatement+ - ElseStatementPart? - FinallyStatement? - ) - | FinallyStatement - ); - scope ExceptStatement = "except" ExceptPattern? ":" ExceptStatementBlock; - ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; - FinallyStatement = "finally" ":" finallyStatement:StatementBlock; - //PEP758 - interface ExceptPattern; - ExpressionListing implements ExceptPattern = (Expression || ",")+; - ParenthesisedExpressionListing implements ExceptPattern = "(" (Expression || ",")+ ")" ("as" Alias)?; - StarredExpressionListing implements ExceptPattern = "*"(Expression || ",")+; - StarredParenthesisedExpressionListing implements ExceptPattern = "*""(" (Expression || ",")+ ")" ("as" Alias)?; - //With open file statement - scope WithStatement implements Statement,ClassStatement = async:"async"? "with" (WithStatementContents || ",")+ ":" StatementBlock ; - WithStatementContents = Expression ("as" target:Name)? ; // target has slicing/etc - - //Variable related statements and definitions - LocalVariableDeclarationStatement implements Statement = VariableDeclaration STATEMENT_END; - VariableDeclaration implements Variable = Name ( - ((":" TypeAnnotation)? "=" VariableInit) - | - (":" TypeAnnotation) - ); - - GlobalVariableDeclaration implements Statement,ClassStatement = "global" names:(Name || ",")+ STATEMENT_END; - NonLocalVariableDeclaration implements Statement,ClassStatement = "nonlocal" names:(Name || ",")+ STATEMENT_END; - MultiVariableDeclaration implements Statement,ClassStatement = (Name || ",")+ ","? "=" (Expression || "," )+ ","? STATEMENT_END; - ParenMultiVariableDeclaration implements Statement,ClassStatement = "(" (Name || ",")+ ","? ")" "=" (Expression || "," )+ ","? STATEMENT_END; - interface VariableInit ; - SimpleInit implements VariableInit = Expression ; - - //function declaration statement and function argument definitions - interface FunctionDeclaration extends Function = Name ; - SimpleFunctionDeclaration implements FunctionDeclaration, Statement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" FunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" + ExceptStatement+ + ElseStatementPart? + FinallyStatement? + ) + | FinallyStatement + ) + ; + scope ExceptStatement = "except" ExceptPattern? ":" ExceptStatementBlock; + ExceptStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; + FinallyStatement = "finally" ":" finallyStatement:StatementBlock; + + //PEP758 + interface ExceptPattern; + AliasedSingleExpressionPattern implements ExceptPattern = Expression ("as" Alias)?; + ExpressionListing implements ExceptPattern = (Expression || ",")+; + ParenthesisedExpressionListing implements ExceptPattern = "(" (Expression || ",")+ ")" ("as" Alias)?; + StarredExpressionListing implements ExceptPattern = "*"(Expression || ",")+; + StarredParenthesisedExpressionListing implements ExceptPattern = "*""(" (Expression || ",")+ ")" ("as" Alias)?; + + //With open file statement + scope WithStatement implements Statement,ClassStatement = async:"async"? "with" (WithStatementContents || ",")+ ":" + StatementBlock ; + + WithStatementContents = Expression ("as" target:Name)? ; // target has slicing/etc + + //Variable related statements and definitions + LocalVariableDeclarationStatement implements Statement = VariableDeclaration STATEMENT_END; + + VariableDeclaration implements Variable = Name ( + ((":" TypeAnnotation)? "=" VariableInit) + | + (":" TypeAnnotation) + ); + + GlobalVariableDeclaration implements Statement,ClassStatement = "global" names:(Name || ",")+ STATEMENT_END; + NonLocalVariableDeclaration implements Statement,ClassStatement = "nonlocal" names:(Name || ",")+ STATEMENT_END; + MultiVariableDeclaration implements Statement,ClassStatement = (Name || ",")+ ","? "=" (Expression || "," )+ ","? STATEMENT_END; + ParenMultiVariableDeclaration implements Statement,ClassStatement = "(" (Name || ",")+ ","? ")" "=" (Expression || "," )+ ","? STATEMENT_END; + + interface VariableInit ; + + SimpleInit implements VariableInit = Expression ; + + //function declaration statement and function argument definitions + interface FunctionDeclaration extends Function = Name ; + + SimpleFunctionDeclaration implements FunctionDeclaration, Statement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? "(" FunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock ; - FunctionParameters = (FunctionParameter || ",")* ","?; - interface FunctionParameter; - SimpleFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)?; - OptionalFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)? "=" Expression ; - VarArgFunctionParameter implements FunctionParameter, Variable = "*" Name GenericsAnnotation? (":" TypeAnnotation)?; - KWArgFunctionParameter implements FunctionParameter, Variable = "**" Name GenericsAnnotation? (":" TypeAnnotation)?; - StarFunctionParameter implements FunctionParameter = "*"; - @Override - Arguments = "(" - (Argument || ",")* - ","? - ")"; - interface Argument; - NormalArgument implements Argument = Expression; - NamedArgument implements Argument = paramName:Name "=" Expression; - - //Behaviour statements - ReturnStatement implements Statement = "return" (Expression || ",")* ","? STATEMENT_END; - YieldStatement implements Statement = "yield" (Expression || ",")* ","? STATEMENT_END; - YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; //PEP380 - RaiseStatement implements Statement,ClassStatement = "raise" (Expression ("from" Name)?)? STATEMENT_END; - - //Typing related statements - TypeDeclaration implements Statement = Expression ":" TypeAnnotation STATEMENT_END; - symbol TypeRuleStatement implements Statement,ClassStatement = key("type") Name GenericsAnnotation? "=" Expression STATEMENT_END; - ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; - - //Match statement - MatchStatement implements Statement,ClassStatement = key("match") Expression ":" MatchBlock; - scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; - scope CaseStatement = key("case") (Expression || "|")+ ("as" Alias)? ("if" condition:Expression)? ":" CaseStatementBlock; - CaseStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; - - //Another Statements - DeleteStatement implements Statement,ClassStatement= "del" (Expression || ",")+ ","? STATEMENT_END; - LiteralStatement implements ClassStatement = Literal STATEMENT_END; - EmptyStatement implements Statement, ClassStatement = STATEMENT_END; - PyDecorator = "@" Expression STATEMENT_END; + FunctionParameters = (FunctionParameter || ",")* ","?; + + interface FunctionParameter; + SimpleFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)?; + OptionalFunctionParameter implements FunctionParameter, Variable = Name GenericsAnnotation? (":" TypeAnnotation)? "=" Expression ; + VarArgFunctionParameter implements FunctionParameter, Variable = "*" Name GenericsAnnotation? (":" TypeAnnotation)?; + KWArgFunctionParameter implements FunctionParameter, Variable = "**" Name GenericsAnnotation? (":" TypeAnnotation)?; + StarFunctionParameter implements FunctionParameter = "*"; + + @Override + Arguments = "(" + (Argument || ",")* + ","? + ")"; + interface Argument; + NormalArgument implements Argument = Expression; + NamedArgument implements Argument = paramName:Name "=" Expression; + + //Behaviour statements + ReturnStatement implements Statement = "return" (Expression || ",")* ","? STATEMENT_END; + YieldStatement implements Statement = "yield" (Expression || ",")* ","? STATEMENT_END; + YieldFromStatement implements Statement = "yield" "from" Expression STATEMENT_END; //PEP380 + RaiseStatement implements Statement,ClassStatement = "raise" (Expression ("from" Name)?)? STATEMENT_END; + + //Typing related statements + TypeDeclaration implements Statement = Expression ":" TypeAnnotation STATEMENT_END; + symbol TypeRuleStatement implements Statement,ClassStatement = key("type") Name GenericsAnnotation? "=" Expression STATEMENT_END; + ExpressionStatement implements Statement = Expression ("," Expression)* STATEMENT_END; + + //Match statement + MatchStatement implements Statement,ClassStatement = key("match") Expression ":" MatchBlock; + scope MatchBlock = BLOCK_START CaseStatement* BLOCK_END; + scope CaseStatement = key("case") CasePattern ("if" condition:Expression)? ":" CaseStatementBlock; + CaseStatementBlock = BLOCK_START Statement* BLOCK_END | Statement; + + // https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-patterns + CasePattern = OpenSequencePattern | Pattern; + Pattern = AsPattern | OrPattern; + AsPattern = OrPattern "as" Alias; + OrPattern = (ClosedPattern|| "|")+; + ClosedPattern = PyQualifiedName | "(" Pattern ")" | SequencePattern | Expression; + + SequencePattern = ("[" OpenSequencePattern? "]") | ("(" OpenSequencePattern? ")"); + OpenSequencePattern = (MaybeStarredPattern || ",")+ ","?; + MaybeStarredPattern = ("*"? Name) | Pattern; + + //Other Statements + DeleteStatement implements Statement,ClassStatement= "del" (Expression || ",")+ ","? STATEMENT_END; + LiteralStatement implements ClassStatement = Literal STATEMENT_END; + EmptyStatement implements Statement, ClassStatement = STATEMENT_END; + PyDecorator = "@" Expression STATEMENT_END; /*====================================== Expressions ======================================*/ - //Aliased Expression only allowed in Case Statements (Checked with CoCo). - AliasedExpression implements Expression = Expression "as" alias:Alias; - symbol Alias = Name; - //Spreadlist expression - SpreadListExpression implements Expression = "*" Expression; - SpreadMappingExpression implements Expression = "**" Expression; - splittoken "**"; - - // ternary-operator expression - TernaryOperatorExpression implements Expression <200> = thenExpression:Expression ( "if" condition:Expression - "else" elseExpression:Expression )+ ; - - //mathematical expression - IntegerDivisionExpression implements Expression <165>, InfixExpression = left:Expression operator:"//" right:Expression ; - IntegerPowExpression implements Expression <195>, InfixExpression = left:Expression operator:"**" right:Expression ; - MatrixMultiplicationExpression implements Expression <200>, InfixExpression = left:Expression operator:"@" right:Expression; - - //logical expressions - AndExpression implements Expression <120>, InfixExpression = left:Expression operator:"and" right:Expression; - OrExpression implements Expression <117>, InfixExpression = left:Expression operator:"or" right:Expression; - NotExpression implements Expression <10> = "not" Expression ; - IsExpression implements Expression <130>, InfixExpression = left:Expression operator:"is" right:Expression; - InExpression implements Expression <195>, InfixExpression = left:Expression operator:"in" right:Expression; - NotInExpression implements Expression <195>, InfixExpression = left:Expression operator:"not" "in" right:Expression; // TODO: set operator to "not in" programmatically - - //Bitwise expressions - BitwiseAndExpression implements Expression <120>, InfixExpression = left:Expression operator:"&" right:Expression; - BitwiseOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"|" right:Expression; - BitwiseXOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"^" right:Expression; - BitwiseLeftShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:"<<" right:Expression; - BitwiseRightShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:">>" right:Expression; - BitwiseOnesComplimentExpression implements Expression <120> = "~" Expression; - - // lambda expression - scope LambdaExpression implements Expression = "lambda" FunctionParameters ":" Expression; - AppliedLambdaExpression implements Expression = "(" LambdaExpression ")" "(" Expression ")"; - - //Assignment expressions - AnnotatedAssignmentExpression implements Expression <60> = - left:Expression - ":" annotated: TypeAnnotation - "=" right:Expression ","?; - @Override - AssignmentExpression implements Expression <60> = - left:Expression - operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" - | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] - right:Expression ","?; - UnpackingAssignmentExpression implements Expression <60> = - "(" left:Expression ("," left:Expression)* ","? ")" "=" right: Expression; - - // 6.3.3 - Slicing - IndexExpression implements Expression = Expression "[" (IndexExpressionInner || ",")+ tuple:","? "]"; - - // slice_item - interface IndexExpressionInner; - SimpleIndex implements IndexExpressionInner = Expression; - ProperSlice implements IndexExpressionInner = lower:Expression? ":" upper:Expression? (":" stride:Expression?)?; - - // Walrus operator - PyAssignmentExpression implements Expression = variable:Name operator:":=" right:Expression; - - //Await expression - AwaitExpression implements Expression = "await" Expression; - - //List/Set/Dict comprehension - ListComprehensionExpression implements Expression = "[" Expression "for" ForControl GeneratorFilter* "]"; - SetComprehensionExpression implements Expression = "{" Expression "for" ForControl GeneratorFilter* "}"; - DictComprehensionExpression implements Expression = "{" Name ":" Expression "for" ForControl GeneratorFilter* "}"; - GeneratorExpression implements Expression = Expression "for" ForControl GeneratorFilter? ; - GeneratorFilter = "if" condition:Expression; + symbol Alias = Name; + + //Spreadlist expression + SpreadListExpression implements Expression = "*" Expression; + SpreadMappingExpression implements Expression = "**" Expression; + splittoken "**"; + + // ternary-operator expression + TernaryOperatorExpression implements Expression <200> = thenExpression:Expression ( "if" condition:Expression + "else" elseExpression:Expression )+ ; + + //mathematical expression + IntegerDivisionExpression implements Expression <165>, InfixExpression = left:Expression operator:"//" right:Expression ; + IntegerPowExpression implements Expression <195>, InfixExpression = left:Expression operator:"**" right:Expression ; + MatrixMultiplicationExpression implements Expression <200>, InfixExpression = left:Expression operator:"@" right:Expression; + + //logical expressions + AndExpression implements Expression <120>, InfixExpression = left:Expression operator:"and" right:Expression ; + OrExpression implements Expression <117>, InfixExpression = left:Expression operator:"or" right:Expression ; + NotExpression implements Expression <10> = "not" Expression ; + IsExpression implements Expression <130>, InfixExpression = left:Expression operator:"is" right:Expression ; + InExpression implements Expression <195>, InfixExpression = left:Expression operator:"in" right:Expression ; + NotInExpression implements Expression <195>, InfixExpression = left:Expression operator:"not" "in" right:Expression; // TODO: set operator to "not in" programmatically + + //Bitwise expressions + BitwiseAndExpression implements Expression <120>, InfixExpression = left:Expression operator:"&" right:Expression; + BitwiseOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"|" right:Expression; + BitwiseXOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"^" right:Expression; + BitwiseLeftShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:"<<" right:Expression; + BitwiseRightShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:">>" right:Expression; + + BitwiseOnesComplimentExpression implements Expression <120> = "~" Expression; + + // lambda expression + scope LambdaExpression implements Expression = "lambda" FunctionParameters ":" Expression ; + AppliedLambdaExpression implements Expression = "(" LambdaExpression ")" "(" Expression ")" ; + + //Assignment expressions + AnnotatedAssignmentExpression implements Expression <60> = + left:Expression + ":" annotated: TypeAnnotation + "=" right:Expression ","?; + @Override + AssignmentExpression implements Expression <60> = + left:Expression + operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|=" + | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="] + right:Expression ","?; + + UnpackingAssignmentExpression implements Expression <60> = + "(" left:Expression ("," left:Expression)* ","? ")" "=" + right: Expression; + + // 6.3.3 - Slicing + IndexExpression implements Expression = Expression "[" (IndexExpressionInner || ",")+ tuple:","? "]"; + + // slice_item + interface IndexExpressionInner; + + SimpleIndex implements IndexExpressionInner = Expression; + ProperSlice implements IndexExpressionInner = lower:Expression? ":" upper:Expression? (":" stride:Expression?)?; + + // Walrus operator + PyAssignmentExpression implements Expression = variable:Name operator:":=" right:Expression; + + //Await expression + AwaitExpression implements Expression = "await" Expression; + + // List/Set/Dict comprehension + ListComprehensionExpression implements Expression = "[" Expression "for" ForControl GeneratorFilter* "]"; + SetComprehensionExpression implements Expression = "{" Expression "for" ForControl GeneratorFilter* "}"; + DictComprehensionExpression implements Expression = "{" Name ":" Expression "for" ForControl GeneratorFilter* "}"; + GeneratorExpression implements Expression = Expression "for" ForControl GeneratorFilter? ; + + GeneratorFilter = "if" condition:Expression; /*===========================Classes======================================*/ - // class symbol - interface scope symbol PythonClass = Name ; + // class symbol + interface scope symbol PythonClass = Name ; + + // class declaration statement + ClassDeclaration implements PythonClass, Statement, ClassStatement = PyDecorator* "class" Name GenericsAnnotation? + ( "(" ((superClasses:PyQualifiedName TypeAnnotation? | arguments:NamedArgument) ","?)* ")" )? ":" ClassStatementBlock ; + + interface ClassStatement; + + ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement; + ClassStatementBlockBody = ClassStatement+; + ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? + "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; - // class declaration statement - ClassDeclaration implements PythonClass, Statement, ClassStatement = PyDecorator* "class" Name GenericsAnnotation? - ( "(" ((superClasses:PyQualifiedName TypeAnnotation? | arguments:NamedArgument) ","?)* ")" )? ":" ClassStatementBlock ; + ClassFunctionParameters = (FunctionParameter || ",")* ","?; - interface ClassStatement; - ClassStatementBlock = (BLOCK_START ClassStatementBlockBody BLOCK_END)|ClassStatement; - ClassStatementBlockBody = ClassStatement+; - ClassFunctionDeclaration implements FunctionDeclaration, ClassStatement = PyDecorator* async:"async"? "def" Name GenericsAnnotation? - "(" ClassFunctionParameters ")" ("->" returnType:TypeAnnotation)? ":" StatementBlock; + ClassAttributes implements ClassStatement = VariableDeclaration STATEMENT_END; - ClassFunctionParameters = (FunctionParameter || ",")* ","?; - ClassAttributes implements ClassStatement = VariableDeclaration STATEMENT_END; - ClassCommentStatement implements ClassStatement = MultiLineStringLiteral STATEMENT_END; + ClassCommentStatement implements ClassStatement = MultiLineStringLiteral STATEMENT_END; /*====================================== Type Annotations ======================================*/ - interface TypeAnnotation; - GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation; - StringTypeAnnotation implements TypeAnnotation = StringLiteralPython; - TupleTypeAnnotation implements TypeAnnotation = "(" (TypeAnnotation || ",")+ ","? ")"; - QualifiedTypeAnnotation implements TypeAnnotation = type:PyQualifiedName ("[" typeParams:(TypeAnnotation || ",")* "]")?; - AlternativeTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "|" rhs:TypeAnnotation; - CommaTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "," rhs:TypeAnnotation ","?; - ListTypeAnnotation implements TypeAnnotation = "[" TypeAnnotation "]"; - EllipsisTypeAnnotation implements TypeAnnotation = "..."; - ComplexTypeAnnotation implements TypeAnnotation <200> = Expression; + interface TypeAnnotation; + GenericTypeAnnotation implements TypeAnnotation = TypeAnnotation GenericsAnnotation; + StringTypeAnnotation implements TypeAnnotation = StringLiteralPython; + TupleTypeAnnotation implements TypeAnnotation = "(" (TypeAnnotation || ",")+ ","? ")"; + QualifiedTypeAnnotation implements TypeAnnotation = type:PyQualifiedName ("[" typeParams:(TypeAnnotation || ",")* "]")?; + AlternativeTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "|" rhs:TypeAnnotation; + CommaTypeAnnotation implements TypeAnnotation = lhs:TypeAnnotation "," rhs:TypeAnnotation ","?; + ListTypeAnnotation implements TypeAnnotation = "[" TypeAnnotation "]"; + EllipsisTypeAnnotation implements TypeAnnotation = "..."; + ComplexTypeAnnotation implements TypeAnnotation <200> = Expression; /*====================================== Generics ======================================*/ - GenericsAnnotation = "[" Generics? "]"; - Generics = (Generic || ",")+ ; - Generic = TypeAnnotation (":" TypeAnnotation)?; + GenericsAnnotation = "[" Generics? "]"; + Generics = (Generic || ",")+ ; + Generic = TypeAnnotation (":" TypeAnnotation)?; } \ No newline at end of file diff --git a/src/main/java/de/monticore/python/_cocos/CaseStatementVisitor.java b/src/main/java/de/monticore/python/_cocos/CaseStatementVisitor.java deleted file mode 100644 index 691911b..0000000 --- a/src/main/java/de/monticore/python/_cocos/CaseStatementVisitor.java +++ /dev/null @@ -1,36 +0,0 @@ -package de.monticore.python._cocos; - -import de.monticore.python._ast.*; -import de.monticore.python._visitor.PythonVisitor2; - -public class CaseStatementVisitor implements PythonVisitor2 { - private boolean inCaseStatement = false; - private int numberOfViolations = 0; - - @Override - public void visit(de.monticore.python._ast.ASTCaseStatement node){ - this.inCaseStatement = true; - } - - @Override - public void endVisit(de.monticore.python._ast.ASTCaseStatement node){ - this.inCaseStatement = false; - } - - @Override - public void visit(de.monticore.python._ast.ASTAliasedExpression node){ - if(!inCaseStatement){ - numberOfViolations++; - } - } - public int getNumberOfViolations (){ - return numberOfViolations; - } - public boolean violationsFound(){ - if(numberOfViolations>0) { - return true; - } else { - return false; - } - } -} \ No newline at end of file diff --git a/src/main/java/de/monticore/python/_cocos/ExpressionsCorrectlyAliased.java b/src/main/java/de/monticore/python/_cocos/ExpressionsCorrectlyAliased.java deleted file mode 100644 index 013fd76..0000000 --- a/src/main/java/de/monticore/python/_cocos/ExpressionsCorrectlyAliased.java +++ /dev/null @@ -1,19 +0,0 @@ -package de.monticore.python._cocos; -import de.monticore.python._ast.ASTPythonScript; -import de.monticore.python.PythonMill; -import de.se_rwth.commons.logging.Log; - -public class ExpressionsCorrectlyAliased implements PythonASTPythonScriptCoCo { - CaseStatementVisitor visitor = new CaseStatementVisitor(); - - @Override - public void check(de.monticore.python._ast.ASTPythonScript node){ - CaseStatementVisitor visitor = new CaseStatementVisitor(); - de.monticore.python._visitor.PythonTraverser traverser = PythonMill.traverser(); - traverser.add4Python(visitor); - node.accept(traverser); - if(visitor.violationsFound()){ - Log.error("In "+visitor.getNumberOfViolations()+" cases an alias was used in an unintended way, in example: var1 = var2 as var 3 "); - } - } -} \ No newline at end of file