Supporting more python

knownToBeUnsupported.md

@@ -0,0 +1,32 @@
+# Currently known to be unsupported
+
+## Tuples without parentheses
+The usage of tuples that are not parenthesized is not supported as these causes issues with antlr. (See https://www.geeksforgeeks.org/python/when-are-parentheses-required-around-a-tuple-in-python/ for unparenthesized tuples)
+ We define the rules:
+
+ ```TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ;```
+ ```SimpleInit implements VariableInit = Expression ;```
+ and ExpressionBasis.mc4 ```LiteralExpression implements Expression <340> = Literal;```
+
+ Thereby without the parentheses we would cause left recursion which is also reachable by ``LiteralStatement implements ClassStatement = Literal STATEMENT_END;```.
+
+## Unicode names
+ Further for now only names with latin letters are permitted, in python another chars are allowed as well, see https://docs.python.org/3/reference/lexical_analysis.html#identifiers. 
+ Prototyping with unicode names have passed the parser tests, further testing needs to be done.
+
+```  @Override
+ token Name =
+            ( UnicodeChar | '_' | '$' )
+            ( UnicodeChar | '_' | '0'..'9' | '$' )*;
+ // Latin,Greek,Coptic,Cyrillic,Armenian
+ fragment token UnicodeChar = 'a'..'z'
+                                  |'A'..'Z'
+                                  |'\u00C0'..'\u00D6'
+                                  |'\u00D8'..'\u00F6'
+                                  |'\u00F8'..'\u02AF'
+                                  |'\u0370'..'\u0373'
+                                  |'\u0376' | '\u0377' | '\u037F' | '\u0386'
+                                  |'\u0386'..'\u03E1'
+                                  |'\u03E2'..'\u0481'
+                                  |'\u048A'..'\u0588';
+```

src/main/grammars/de/monticore/MultilineString.mc4

@@ -5,5 +5,6 @@ package de.monticore;
 // This can be fixed by adding another token, which must be defined before the String token.
 // Thus, this grammar must be used before MCCommonLiterals, which most langauges use.
 component grammar MultilineString {
-  token DoubleQuoteMultilineStringDelimiter = '"' '"' '"';
+  token DoubleQuoteMultilineFStringDelimiter = ('f'|'F') '"' '"' '"';
+  token DoubleQuoteMultilineStringDelimiter =  '"' '"' '"';
 }

src/main/grammars/de/monticore/PythonBasis.mc4

@@ -0,0 +1,265 @@
+/* (c) https://github.com/MontiCore/monticore */
+package de.monticore;
+
+import de.monticore.MultilineString;
+import de.monticore.expressions.*;
+import de.monticore.literals.MCCommonLiterals;
+import de.monticore.symbols.OOSymbols;
+
+grammar PythonBasis extends MultilineString,
+                               MCCommonLiterals,
+                               CommonExpressions,
+                               AssignmentExpressions,
+                               OOSymbols{
+
+  /*====================================== Tokens ======================================*/
+  @Override
+  token WS = (' ' | '\t' | '\r' | '\n' ) : -> channel(HIDDEN);
+  @Override
+  token SL_COMMENT = "#" (~('\n' | '\r' ))* : -> channel(HIDDEN);
+  token ByteOrderMark = '\uFEFF' : -> skip;
+
+  /**
+  * The following utf-8 symbols are used to parse the Python files without having to add whitespace to the Grammar.
+  * In a preprocessing step code blocks are denoted with \u204f = ⦃ and \u2984 = ⦄, and lines are ended with \u204f = ⁏
+  */
+  token BLOCK_START = '\u2983';
+  token BLOCK_END = '\u2984';
+  token STATEMENT_END = ';'? '\u204f' | ';' '\u204f'?;
+
+  // Will be filtered out by the WhitespacePreprocessingTokenSource
+  // Used break a line without finishing the statement
+  token CONTINUE_LINE_TOKEN = '\\' '\r'? '\n';
+
+  // === string tokens for python ===
+  //Often (mis)used as a multiline comment but can also be used as a string literal, thus we can not skip it
+  token MultiLineStringToken = ((("\'\'\'") .*? ("\'\'\'")) | ((DoubleQuoteMultilineStringDelimiter .*? DoubleQuoteMultilineStringDelimiter)));
+  token MultiLineFStringToken = ((("\'\'\'") .*? ("\'\'\'")) |((DoubleQuoteMultilineFStringDelimiter .*? DoubleQuoteMultilineStringDelimiter)));
+
+  //Double quoted Strings "Text"
+  @Override
+  token String = '"' (StringDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));};
+
+  fragment token StringDQCharactersPython
+    = (StringDQCharacterPython)+;
+  fragment token StringDQCharacterPython
+    = ~ ('"'| '\\' | '\n') | PythonEscapeSequence;
+
+  //Single quoted Strings 'Text'
+  token StringPython = '\'' (StringSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));};
+
+  fragment token StringSQCharactersPython
+    = (StringSQCharacterPython)+;
+  fragment token StringSQCharacterPython
+    = ~ ('\''| '\\' | '\n') | PythonEscapeSequence;
+
+  //Escape in strings "\n"
+  fragment token PythonEscapeSequence = '\\' .;
+
+  //Double and single quoted strings with an f modifier f'text1{exp}text2', separate definition to allow more expressions.
+  token FSQStringPython
+    = ('f'|'F') '\'' (StringFSQCharactersPython)? '\'' : {setText(getText().substring(1, getText().length() - 1));};
+  token FDQStringPython
+    = ('f'|'F') '"' (StringFDQCharactersPython)? '"' : {setText(getText().substring(1, getText().length() - 1));};
+
+  fragment token StringFSQCharactersPython
+    = (StringFSQCharacterPython)+;
+  fragment token StringFDQCharactersPython
+    = (StringFDQCharacterPython)+;
+
+  fragment token StringFSQCharacterPython
+    = ~ ('\''| '\\')| PythonEscapeSequence;
+  fragment token StringFDQCharacterPython
+    = ~ ('"'| '\\') | PythonEscapeSequence;
+
+  // === number tokens for python ===
+
+  token FloatWithExponent = (DigitsPart | PyFloat) ('e'|'E') ('+' | '-')? DigitsPart;
+  FloatWithExponentLiteral implements NumericLiteral <100> = FloatWithExponent;
+
+  token PyFloat = DigitsPart? '.' DigitsPart | DigitsPart '.';
+  PyFloatLiteral implements NumericLiteral <200> = PyFloat;
+
+  // PEP 515
+  token DigitsPart = Digit ('_'? Digit)*;
+
+  token HexNumberToken = '0' 'x' ('0'..'9' | 'a'..'f' | 'A'..'F')+;
+  // PEP 515: Underscores in Numeric Literals
+  @Override
+  token Digits
+    = Digit ('_'?  Digit)*; // technically the first digit must be nonzero except for 0(_0)*
+
+  /*====================================== Literals ======================================*/
+
+  HexNumberLiteral implements NumericLiteral <100> = HexNumberToken;
+  ImaginaryNumberLiteral implements NumericLiteral <100> = {noSpace(2)}? (DigitsPart| Digits | PyFloat) key("j");
+
+  //Literals for Datastructures
+  ArrayLiteral implements Literal = "[" (VariableInit || ",")* ","? "]" ;
+  TupleLiteral implements Literal = "(" (VariableInit || ",")* ","? ")" ;
+  DictLiteral implements Literal = "{" (DictEntry || ",")* ","? "}" ;
+  SetLiteral implements Literal = "{" (Expression || ",")* ","? "}" ;
+
+  DictEntry = key:VariableInit ":" value:VariableInit | SpreadMappingExpression;
+
+  //Literals and helper-definitions regarding Strings
+  MultiLineStringLiteral implements Literal = (StringModifier)? MultiLineStringToken;
+  MultiLineFStringLiteral implements Literal = (StringModifier)? MultiLineFStringToken;
+
+  //Char is necessary because single quoted characters will be recognized as char tokens not string tokens,
+  // as char is imported from MCCommonLiterals.mc4
+  StringLiteralPython implements Literal, SignedLiteral =
+          (
+              StringModifier?
+              (sourceStrPy:StringPython | sourceStr:String |sourceChar:Char)
+          )
+          | fsource: FStringPython;
+
+  StringsLiteralPython implements Literal <200> = (StringLiteralPython | StringLiteral | MultiLineStringLiteral | MultiLineFStringLiteral | FStringPython)+;
+
+  FStringPython =  (FSQStringPython | FDQStringPython | ("f"|"F") Char);
+  StringModifier = /*{cmpTokenRegEx(1, "(r|b|u|R|B|U)+")}?*/ type:Name;
+
+  // boolean literals for python
+  @Override
+  BooleanLiteral implements Literal, SignedLiteral =
+      source:["True" | "False"];
+
+  // https://docs.python.org/dev/library/constants.html#Ellipsis
+  EllipsisLiteral implements Literal = "...";
+  splittoken "...";
+
+  /*====================================== Expressions ======================================*/
+  //Spreadlist expression
+  SpreadListExpression implements Expression = "*" Expression;
+  SpreadMappingExpression implements Expression = "**" Expression;
+  splittoken "**";
+
+  // ternary-operator expression
+  TernaryOperatorExpression implements Expression <200> = thenExpression:Expression ( "if" condition:Expression
+                                       "else" elseExpression:Expression )+ ;
+
+  //mathematical expression
+  IntegerDivisionExpression implements Expression <165>, InfixExpression = left:Expression  operator:"//" right:Expression ;
+  IntegerPowExpression implements Expression <195>, InfixExpression = left:Expression operator:"**" right:Expression ;
+  MatrixMultiplicationExpression implements Expression <200>, InfixExpression = left:Expression operator:"@" right:Expression;
+
+  //logical expressions
+  AndExpression implements Expression <120>, InfixExpression = left:Expression operator:"and" right:Expression ;
+  OrExpression implements Expression <117>, InfixExpression = left:Expression operator:"or" right:Expression ;
+  NotExpression implements Expression <10> = "not" Expression ;
+  IsExpression implements Expression <130>, InfixExpression = left:Expression operator:"is" right:Expression ;
+  InExpression implements Expression <195>, InfixExpression = left:Expression operator:"in" right:Expression ;
+  NotInExpression implements Expression <195>, InfixExpression = left:Expression operator:"not" "in" right:Expression; // TODO: set operator to "not in" programmatically
+
+  //Bitwise expressions
+  BitwiseAndExpression implements Expression <120>, InfixExpression = left:Expression operator:"&" right:Expression;
+  BitwiseOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"|" right:Expression;
+  BitwiseXOrExpression implements Expression <120>, InfixExpression = left:Expression operator:"^" right:Expression;
+  BitwiseLeftShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:"<<" right:Expression;
+  BitwiseRightShiftExpression implements Expression <120>, InfixExpression = left:Expression operator:">>" right:Expression;
+
+  BitwiseOnesComplimentExpression implements Expression <120> = "~" Expression;
+
+  // lambda expression
+  scope LambdaExpression implements Expression = "lambda" FunctionParameters ":" Expression ;
+  AppliedLambdaExpression implements Expression = "(" LambdaExpression ")" "(" Expression ")" ;
+
+  //Assignment expressions
+  AnnotatedAssignmentExpression implements Expression <60> = <rightassoc>
+      left:Expression
+      ":" annotated: TypeAnnotation
+      "=" right:Expression ","?;
+  @Override
+  AssignmentExpression implements Expression <60> = <rightassoc>
+      left:Expression
+      operator: [ "=" | "+=" | "-=" | "*=" | "/=" | "&=" | "|="
+                | "^=" | ">>=" | ">>>=" | "<<=" | "%=" | "**=" | "@=" | "//="]
+      right:Expression ","?;
+
+  UnpackingAssignmentExpression implements Expression <60> = <rightassoc>
+    "(" left:Expression ("," left:Expression)* ","? ")" "="
+    right: Expression;
+
+ // 6.3.3 - Slicing
+  IndexExpression implements Expression = Expression "[" (IndexExpressionInner || ",")+ tuple:","? "]";
+
+  // slice_item
+  interface IndexExpressionInner;
+
+  SimpleIndex implements IndexExpressionInner = Expression;
+  ProperSlice implements IndexExpressionInner = lower:Expression? ":" upper:Expression? (":" stride:Expression?)?;
+
+  // Walrus operator
+  PyAssignmentExpression implements Expression = variable:Name operator:":=" right:Expression;
+
+  //Await expression
+  AwaitExpression implements Expression = "await" Expression;
+
+  // List/Set/Dict comprehension
+  ListComprehensionExpression implements Expression = "[" Expression "for" ForControl GeneratorFilter* "]";
+  SetComprehensionExpression implements Expression = "{" Expression "for" ForControl GeneratorFilter* "}";
+  DictComprehensionExpression implements Expression = "{" Name ":" Expression "for" ForControl GeneratorFilter* "}";
+  GeneratorExpression implements Expression = Expression "for" ForControl GeneratorFilter? ;
+
+
+  ForControl = ForList "in" ForIterable;
+
+  interface ForDecomposition;
+  ForList = (ForDecomposition || ",")+ ","?;
+  ForVariable implements Variable, ForDecomposition = Name;
+  ForDecompositionParenthesis implements ForDecomposition = "(" ForList? ")";
+  ForDecompositionBrackets implements ForDecomposition = "[" ForList? "]";
+  ForStarredVariable  implements ForDecomposition = "*" ForDecomposition ;
+  ForPyQualifiedName implements ForDecomposition = PyQualifiedName;
+  ForIterable = Expression;
+
+  GeneratorFilter = "if" condition:Expression;
+
+  /*====================================== Type Annotations ======================================*/
+
+  interface TypeAnnotation;
+  GenericTypeAnnotation     implements TypeAnnotation <100> = TypeAnnotation GenericsAnnotation;
+  StringTypeAnnotation      implements TypeAnnotation <100> = StringLiteralPython;
+  TupleTypeAnnotation       implements TypeAnnotation <100> = "(" (TypeAnnotation || ",")+ ","? ")";
+  QualifiedTypeAnnotation   implements TypeAnnotation <200> = type:PyQualifiedName;
+  AlternativeTypeAnnotation implements TypeAnnotation <100> = <rightassoc> lhs:TypeAnnotation "|" rhs:TypeAnnotation;
+  CommaTypeAnnotation       implements TypeAnnotation <60> = <rightassoc> lhs:TypeAnnotation "," rhs:TypeAnnotation ","?;
+  ListTypeAnnotation        implements TypeAnnotation <100> = "[" TypeAnnotation "]";
+  EllipsisTypeAnnotation    implements TypeAnnotation <100> = "...";
+  ComplexTypeAnnotation     implements TypeAnnotation <50> = Expression;
+
+  /*====================================== Generics ======================================*/
+
+  GenericsAnnotation = "[" Generics? "]";
+  Generics = (Generic || ",")+ ;
+  Generic = type:TypeAnnotation (":" TypeAnnotation)?;
+
+  /*====================================== Variables ======================================*/
+
+  interface VariableInit ;
+  SimpleInit implements VariableInit = Expression ;
+
+  //Helper-definition for qualified names in python.
+  PyQualifiedName = (Name || ".")+;
+  astrule PyQualifiedName = method public String joined(){
+    return String.join(".", getNameList());
+  };
+
+  FunctionParameters = (FunctionParameter  || ",")* ","?;
+  interface FunctionParameter;
+  SimpleFunctionParameter   implements FunctionParameter, Variable = Name  (":" TypeAnnotation)?;
+  OptionalFunctionParameter implements FunctionParameter, Variable = Name  (":" TypeAnnotation)? "=" Expression ;
+  VarArgFunctionParameter   implements FunctionParameter, Variable = "*" Name  (":" TypeAnnotation)?;
+  KWArgFunctionParameter    implements FunctionParameter, Variable = "**" Name   (":" TypeAnnotation)?;
+  StarFunctionParameter     implements FunctionParameter = "*";
+
+  @Override
+  Arguments = "("
+    (Argument || ",")*
+     ","?
+  ")";
+  interface Argument;
+  NormalArgument implements Argument = Expression;
+  NamedArgument implements Argument = paramName:Name "=" Expression;
+  }

src/main/java/de/monticore/python/_cocos/PythonFunctionArgumentSizeCoco.java

@@ -5,9 +5,9 @@
 import de.monticore.expressions.commonexpressions._cocos.CommonExpressionsASTCallExpressionCoCo;
 import de.monticore.expressions.expressionsbasis._ast.ASTExpression;
 import de.monticore.expressions.expressionsbasis._ast.ASTNameExpression;
-import de.monticore.python._ast.ASTArgument;
-import de.monticore.python._ast.ASTArguments;
-import de.monticore.python._ast.ASTOptionalFunctionParameter;
+import de.monticore.pythonbasis._ast.ASTArgument;
+import de.monticore.pythonbasis._ast.ASTArguments;
+import de.monticore.pythonbasis._ast.ASTOptionalFunctionParameter;
 import de.monticore.python._symboltable.IPythonScope;
 import de.monticore.symbols.basicsymbols._symboltable.FunctionSymbol;
 import de.se_rwth.commons.logging.Log;

src/main/java/de/monticore/python/_cocos/PythonFunctionDuplicateParameterNameCoco.java

@@ -2,9 +2,9 @@
 
 import de.monticore.python._ast.ASTClassFunctionDeclaration;
 import de.monticore.python._ast.ASTFunctionDeclaration;
-import de.monticore.python._ast.ASTFunctionParameter;
+import de.monticore.pythonbasis._ast.ASTFunctionParameter;
 import de.monticore.python._ast.ASTSimpleFunctionDeclaration;
-import de.monticore.python._util.PythonTypeDispatcher;
+import de.monticore.pythonbasis._util.PythonBasisTypeDispatcher;
 import de.se_rwth.commons.logging.Log;
 
 import java.util.ArrayList;
@@ -26,14 +26,14 @@ public void check(ASTFunctionDeclaration node) {
 			parameters.addAll(((ASTClassFunctionDeclaration) node).getClassFunctionParameters().getFunctionParameterList());
 		}
 
-		PythonTypeDispatcher td = new PythonTypeDispatcher();
+		PythonBasisTypeDispatcher td = new PythonBasisTypeDispatcher();
 
 		for (ASTFunctionParameter parameter : parameters) {
       String name = null;
-			if(td.isBasicSymbolsASTTypeVar(parameter)) {
+		if(td.isBasicSymbolsASTTypeVar(parameter)) {
 				name = td.asBasicSymbolsASTTypeVar(parameter).getName();
-			}else if(td.isPythonASTSimpleFunctionParameter(parameter)){
-        name = td.asPythonASTSimpleFunctionParameter(parameter).getName();
+		}else if(td.isPythonBasisASTSimpleFunctionParameter(parameter)) {
+        name = td.asPythonBasisASTSimpleFunctionParameter(parameter).getName();
       }
       if (name != null) {
         if (names.contains(name)) {