diff --git a/docs/site/dml-language-reference.md b/docs/site/dml-language-reference.md index 264b3c6a2b1..6a26c4caaac 100644 --- a/docs/site/dml-language-reference.md +++ b/docs/site/dml-language-reference.md @@ -2068,10 +2068,12 @@ The following example uses transformapply() with the input matrix a **Table F5**: Frame processing built-in functions -Function | Description | Parameters | Example --------- | ----------- | ---------- | ------- -map() | It will execute the given lambda expression on a frame (cell, row or column wise). | Input: (X <frame>, y <String>, \[margin <int>\])
Output: <frame>.
X is a frame and
y is a String containing the lambda expression to be executed on frame X.
margin - how to apply the lambda expression (0 indicates each cell, 1 - rows, 2 - columns). Output matrix dimensions are always equal to the input. | [map](#map) -tokenize() | Transforms a frame to tokenized frame using specification. Tokenization is valid only for string columns. | Input:
target = <frame>
spec = <json specification>
Outputs: <matrix>, <frame> | [tokenize](#tokenize) +Function | Description | Parameters | Example +-------- |-----------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ------- +map() | It will execute the given lambda expression on a frame (cell, row or column wise). | Input: (X <frame>, y <String>, \[margin <int>\])
Output: <frame>.
X is a frame and
y is a String containing the lambda expression to be executed on frame X.
margin - how to apply the lambda expression (0 indicates each cell, 1 - rows, 2 - columns). Output matrix dimensions are always equal to the input. | [map](#map) +tokenize() | Transforms a frame to tokenized frame using specification. Tokenization is valid only for string columns. | Input:
target = <frame>
spec = <json specification>
Outputs: <matrix>, <frame> | [tokenize](#tokenize) +getNames() | Returns the column names of a frame as a single-row frame. | Input: X <frame>
Output: <frame> | N = getNames(X) +setNames() | Sets the column names of a frame from a single-row frame containing string values. | Input:
X = <frame>
N = <frame>
Output:<frame> | Y = setNames(X, N) #### map diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java index f5719641df7..fa5da4cd91d 100644 --- a/src/main/java/org/apache/sysds/common/Builtins.java +++ b/src/main/java/org/apache/sysds/common/Builtins.java @@ -154,6 +154,7 @@ public enum Builtins { GARCH("garch", true), GAUSSIAN_CLASSIFIER("gaussianClassifier", true), GET_ACCURACY("getAccuracy", true), + GET_NAMES("getNames", false), GET_CATEGORICAL_MASK("getCategoricalMask", false), GLM("glm", true), GLM_PREDICT("glmPredict", true), @@ -310,6 +311,7 @@ public enum Builtins { SELVARTHRESH("selectByVarThresh", true), SEQ("seq", false), SES("ses", true), + SET_NAMES("setNames", false), SYMMETRICDIFFERENCE("symmetricDifference", true), SHAPEXPLAINER("shapExplainer", true), SHERLOCK("sherlock", true), diff --git a/src/main/java/org/apache/sysds/common/Opcodes.java b/src/main/java/org/apache/sysds/common/Opcodes.java index 9a894dde13b..0f971b764c8 100644 --- a/src/main/java/org/apache/sysds/common/Opcodes.java +++ b/src/main/java/org/apache/sysds/common/Opcodes.java @@ -352,6 +352,7 @@ public enum Opcodes { MAPPM("map+*", InstructionType.Binary), MAPMINUSMULT("map-*", InstructionType.Binary), MAPDROPINVALIDLENGTH("mapdropInvalidLength", InstructionType.Binary), + SET_COLNAMES("set_colnames", InstructionType.Binary), MAPGT("map>", InstructionType.Binary), MAPGE("map>=", InstructionType.Binary), diff --git a/src/main/java/org/apache/sysds/common/Types.java b/src/main/java/org/apache/sysds/common/Types.java index c2832aeb8cd..e7282cde00e 100644 --- a/src/main/java/org/apache/sysds/common/Types.java +++ b/src/main/java/org/apache/sysds/common/Types.java @@ -641,7 +641,8 @@ public enum OpOp2 { MINUS1_MULT(false), //1-X*Y GET_CATEGORICAL_MASK(false), // get transformation mask QUANTIZE_COMPRESS(false), //quantization-fused compression - UNION_DISTINCT(false); + UNION_DISTINCT(false), + SET_COLNAMES(false); private final boolean _validOuter; diff --git a/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java index ab0c7993b4e..bb64e23f922 100644 --- a/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java @@ -46,12 +46,13 @@ public class BuiltinFunctionExpression extends DataIdentifier { protected Expression[] _args = null; private Builtins _opcode; - public BuiltinFunctionExpression(ParserRuleContext ctx, Builtins bifop, ArrayList args, String fname) { + public BuiltinFunctionExpression(ParserRuleContext ctx, Builtins bifop, ArrayList args, + String fname) { _opcode = bifop; setCtxValuesAndFilename(ctx, fname); args = expandDnnArguments(args); _args = new Expression[args.size()]; - for(int i=0; i < args.size(); i++) { + for(int i = 0; i < args.size(); i++) { _args[i] = args.get(i).getExpr(); } } @@ -59,7 +60,7 @@ public BuiltinFunctionExpression(ParserRuleContext ctx, Builtins bifop, ArrayLis public BuiltinFunctionExpression(Builtins bifop, Expression[] args, ParseInfo parseInfo) { _opcode = bifop; _args = new Expression[args.length]; - for (int i = 0; i < args.length; i++) { + for(int i = 0; i < args.length; i++) { _args[i] = args[i]; } setParseInfo(parseInfo); @@ -68,7 +69,7 @@ public BuiltinFunctionExpression(Builtins bifop, Expression[] args, ParseInfo pa public BuiltinFunctionExpression(ParserRuleContext ctx, Builtins bifop, Expression[] args, String fname) { _opcode = bifop; _args = new Expression[args.length]; - for(int i=0; i < args.length; i++) { + for(int i = 0; i < args.length; i++) { _args[i] = args[i]; } setCtxValuesAndFilename(ctx, fname); @@ -77,7 +78,7 @@ public BuiltinFunctionExpression(ParserRuleContext ctx, Builtins bifop, Expressi @Override public Expression rewriteExpression(String prefix) { Expression[] newArgs = new Expression[_args.length]; - for (int i = 0; i < _args.length; i++) { + for(int i = 0; i < _args.length; i++) { newArgs[i] = _args[i].rewriteExpression(prefix); } BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(this._opcode, newArgs, this); @@ -99,19 +100,19 @@ public Expression getSecondExpr() { public Expression getThirdExpr() { return (_args.length >= 3 ? _args[2] : null); } - + public Expression getFourthExpr() { return (_args.length >= 4 ? _args[3] : null); } - + public Expression getFifthExpr() { return (_args.length >= 5 ? _args[4] : null); } - + public Expression getSixthExpr() { return (_args.length >= 6 ? _args[5] : null); } - + public Expression getSeventhExpr() { return (_args.length >= 7 ? _args[6] : null); } @@ -120,27 +121,26 @@ public Expression getEighthExpr() { return (_args.length >= 8 ? _args[7] : null); } - - public Expression[] getAllExpr(){ + public Expression[] getAllExpr() { return _args; } - + public Expression getExpr(int i) { return (_args.length > i ? _args[i] : null); } - + @Override - public void validateExpression(MultiAssignmentStatement stmt, HashMap ids, HashMap constVars, boolean conditional) - { - if (this.getFirstExpr() instanceof FunctionCallIdentifier){ + public void validateExpression(MultiAssignmentStatement stmt, HashMap ids, + HashMap constVars, boolean conditional) { + if(this.getFirstExpr() instanceof FunctionCallIdentifier) { raiseValidateError("UDF function call not supported as parameter to built-in function call", false); } - + this.getFirstExpr().validateExpression(ids, constVars, conditional); - Expression [] expr = getAllExpr(); + Expression[] expr = getAllExpr(); if(expr != null && expr.length > 1) { for(int i = 1; i < expr.length; i++) { - if (expr[i] instanceof FunctionCallIdentifier){ + if(expr[i] instanceof FunctionCallIdentifier) { raiseValidateError("UDF function call not supported as parameter to built-in function call", false); } expr[i].validateExpression(ids, constVars, conditional); @@ -148,645 +148,642 @@ public void validateExpression(MultiAssignmentStatement stmt, HashMap 0) { - raiseValidateError("Missing argument for function " + this.getOpCode(), false, - LanguageErrorCodes.INVALID_PARAMETERS); - } - else if(getFifthExpr() != null) { - raiseValidateError("Invalid number of arguments for function " + this.getOpCode().toString().toLowerCase() - + "(). This function only takes 3 or 4 arguments.", false); } - else if(_args.length == 3) { - checkScalarParam(getSecondExpr()); - checkScalarParam(getThirdExpr()); - if(!isPowerOfTwo(((ConstIdentifier) getSecondExpr().getOutput()).getLongValue())) { - raiseValidateError( - "This FFT implementation is only defined for matrices with dimensions that are powers of 2." - + "The window size (2nd argument) is not a power of two", - false, LanguageErrorCodes.INVALID_PARAMETERS); + case IFFT_LINEARIZED: { + Expression expressionTwo = getSecondExpr(); + Expression expressionOne = getFirstExpr(); + + if(expressionOne == null) { + raiseValidateError("The first argument to " + _opcode + " cannot be null.", false, + LanguageErrorCodes.INVALID_PARAMETERS); } - else if(((ConstIdentifier) getSecondExpr().getOutput()) - .getLongValue() <= ((ConstIdentifier) getThirdExpr().getOutput()).getLongValue()) { - raiseValidateError("Overlap can't be larger than or equal to the window size.", false, + else if(expressionOne.getOutput() == null || expressionOne.getOutput().getDim1() == 0 || + expressionOne.getOutput().getDim2() == 0) { + raiseValidateError("The first argument to " + _opcode + " cannot be an empty matrix.", false, LanguageErrorCodes.INVALID_PARAMETERS); } + else if(expressionTwo != null) { + if(expressionTwo.getOutput() == null || expressionTwo.getOutput().getDim1() == 0 || + expressionTwo.getOutput().getDim2() == 0) { + raiseValidateError("The second argument to " + _opcode + + " cannot be an empty matrix. Provide either only a real matrix or a filled real and imaginary one.", + false, LanguageErrorCodes.INVALID_PARAMETERS); + } + } + + checkNumParameters(expressionTwo != null ? 2 : 1); + checkMatrixParam(expressionOne); + if(expressionTwo != null && expressionOne != null) { + checkMatrixParam(expressionTwo); + if(expressionOne.getOutput().getDim1() != expressionTwo.getOutput().getDim1() || + expressionOne.getOutput().getDim2() != expressionTwo.getOutput().getDim2()) + raiseValidateError( + "The real and imaginary part of the provided matrix are of different dimensions.", false); + else if(!isPowerOfTwo(expressionTwo.getOutput().getDim2())) { + raiseValidateError( + "This IFFT_LINEARIZED implementation is only defined for matrices with columns that are powers of 2.", + false, LanguageErrorCodes.INVALID_PARAMETERS); + } + } + else if(expressionOne != null) { + if(!isPowerOfTwo(expressionOne.getOutput().getDim2())) { + raiseValidateError( + "This IFFT_LINEARIZED implementation is only defined for matrices with columns that are powers of 2.", + false, LanguageErrorCodes.INVALID_PARAMETERS); + } + } + + DataIdentifier ifftOut1 = (DataIdentifier) getOutputs()[0]; + DataIdentifier ifftOut2 = (DataIdentifier) getOutputs()[1]; + + ifftOut1.setDataType(DataType.MATRIX); + ifftOut1.setValueType(ValueType.FP64); + ifftOut1.setDimensions(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2()); + ifftOut1.setBlocksize(getFirstExpr().getOutput().getBlocksize()); + + ifftOut2.setDataType(DataType.MATRIX); + ifftOut2.setValueType(ValueType.FP64); + ifftOut2.setDimensions(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2()); + ifftOut2.setBlocksize(getFirstExpr().getOutput().getBlocksize()); + + break; } - else if(_args.length == 4) { - checkMatrixParam(getSecondExpr()); - checkScalarParam(getThirdExpr()); - checkScalarParam(getFourthExpr()); - if(!isPowerOfTwo(((ConstIdentifier) getThirdExpr().getOutput()).getLongValue())) { + case STFT: { + checkMatrixParam(getFirstExpr()); + + if((getFirstExpr() == null || getSecondExpr() == null || getThirdExpr() == null) && _args.length > 0) { + raiseValidateError("Missing argument for function " + this.getOpCode(), false, + LanguageErrorCodes.INVALID_PARAMETERS); + } + else if(getFifthExpr() != null) { raiseValidateError( - "This FFT implementation is only defined for matrices with dimensions that are powers of 2." - + "The window size (3rd argument) is not a power of two", - false, LanguageErrorCodes.INVALID_PARAMETERS); + "Invalid number of arguments for function " + this.getOpCode().toString().toLowerCase() + + "(). This function only takes 3 or 4 arguments.", false); } - else if(getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1() || - getFirstExpr().getOutput().getDim2() != getSecondExpr().getOutput().getDim2()) { - raiseValidateError("The real and imaginary part of the provided matrix are of different dimensions.", - false); + else if(_args.length == 3) { + checkScalarParam(getSecondExpr()); + checkScalarParam(getThirdExpr()); + if(!isPowerOfTwo(((ConstIdentifier) getSecondExpr().getOutput()).getLongValue())) { + raiseValidateError( + "This FFT implementation is only defined for matrices with dimensions that are powers of 2." + + "The window size (2nd argument) is not a power of two", false, + LanguageErrorCodes.INVALID_PARAMETERS); + } + else if(((ConstIdentifier) getSecondExpr().getOutput()).getLongValue() <= + ((ConstIdentifier) getThirdExpr().getOutput()).getLongValue()) { + raiseValidateError("Overlap can't be larger than or equal to the window size.", false, + LanguageErrorCodes.INVALID_PARAMETERS); + } } - else if(((ConstIdentifier) getThirdExpr().getOutput()) - .getLongValue() <= ((ConstIdentifier) getFourthExpr().getOutput()).getLongValue()) { - raiseValidateError("Overlap can't be larger than or equal to the window size.", false, - LanguageErrorCodes.INVALID_PARAMETERS); + else if(_args.length == 4) { + checkMatrixParam(getSecondExpr()); + checkScalarParam(getThirdExpr()); + checkScalarParam(getFourthExpr()); + if(!isPowerOfTwo(((ConstIdentifier) getThirdExpr().getOutput()).getLongValue())) { + raiseValidateError( + "This FFT implementation is only defined for matrices with dimensions that are powers of 2." + + "The window size (3rd argument) is not a power of two", false, + LanguageErrorCodes.INVALID_PARAMETERS); + } + else if(getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1() || + getFirstExpr().getOutput().getDim2() != getSecondExpr().getOutput().getDim2()) { + raiseValidateError( + "The real and imaginary part of the provided matrix are of different dimensions.", false); + } + else if(((ConstIdentifier) getThirdExpr().getOutput()).getLongValue() <= + ((ConstIdentifier) getFourthExpr().getOutput()).getLongValue()) { + raiseValidateError("Overlap can't be larger than or equal to the window size.", false, + LanguageErrorCodes.INVALID_PARAMETERS); + } } - } - // setup output properties - DataIdentifier stftOut1 = (DataIdentifier) getOutputs()[0]; - DataIdentifier stftOut2 = (DataIdentifier) getOutputs()[1]; + // setup output properties + DataIdentifier stftOut1 = (DataIdentifier) getOutputs()[0]; + DataIdentifier stftOut2 = (DataIdentifier) getOutputs()[1]; - // Output1 - stft Values - stftOut1.setDataType(DataType.MATRIX); - stftOut1.setValueType(ValueType.FP64); - stftOut1.setDimensions(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2()); - stftOut1.setBlocksize(getFirstExpr().getOutput().getBlocksize()); + // Output1 - stft Values + stftOut1.setDataType(DataType.MATRIX); + stftOut1.setValueType(ValueType.FP64); + stftOut1.setDimensions(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2()); + stftOut1.setBlocksize(getFirstExpr().getOutput().getBlocksize()); - // Output2 - stft Vectors - stftOut2.setDataType(DataType.MATRIX); - stftOut2.setValueType(ValueType.FP64); - stftOut2.setDimensions(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2()); - stftOut2.setBlocksize(getFirstExpr().getOutput().getBlocksize()); + // Output2 - stft Vectors + stftOut2.setDataType(DataType.MATRIX); + stftOut2.setValueType(ValueType.FP64); + stftOut2.setDimensions(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2()); + stftOut2.setBlocksize(getFirstExpr().getOutput().getBlocksize()); - break; - } - case REMOVE: { - checkNumParameters(2); - checkListParam(getFirstExpr()); - - // setup output properties - DataIdentifier out1 = (DataIdentifier) getOutputs()[0]; - DataIdentifier out2 = (DataIdentifier) getOutputs()[1]; - - // Output1 - list after removal - long nrow = getFirstExpr().getOutput().getDim1() > 0 ? - getFirstExpr().getOutput().getDim1() + 1 : -1; - out1.setDataType(DataType.LIST); - out1.setValueType(getFirstExpr().getOutput().getValueType()); - out1.setDimensions(nrow, 1); - out1.setBlocksize(getFirstExpr().getOutput().getBlocksize()); - - // Output2 - list of removed element - out2.setDataType(DataType.LIST); - out2.setValueType(getFirstExpr().getOutput().getValueType()); - out2.setDimensions(1, 1); - out2.setBlocksize(getFirstExpr().getOutput().getBlocksize()); - - break; - } - case SVD: - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - - long minMN = Math.min(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2()); - - // setup output properties - DataIdentifier svdOut1 = (DataIdentifier) getOutputs()[0]; - DataIdentifier svdOut2 = (DataIdentifier) getOutputs()[1]; - DataIdentifier svdOut3 = (DataIdentifier) getOutputs()[2]; - - // Output 1 - svdOut1.setDataType(DataType.MATRIX); - svdOut1.setValueType(ValueType.FP64); - svdOut1.setDimensions(getFirstExpr().getOutput().getDim1(), minMN); - svdOut1.setBlocksize(getFirstExpr().getOutput().getBlocksize()); - - // Output 2 - svdOut2.setDataType(DataType.MATRIX); - svdOut2.setValueType(ValueType.FP64); - svdOut2.setDimensions(minMN, minMN); - svdOut2.setBlocksize(getFirstExpr().getOutput().getBlocksize()); - - // Output 3 - svdOut3.setDataType(DataType.MATRIX); - svdOut3.setValueType(ValueType.FP64); - svdOut3.setDimensions(getFirstExpr().getOutput().getDim2(), minMN); - svdOut3.setBlocksize(getFirstExpr().getOutput().getBlocksize()); - - break; - - case COMPRESS: - if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_COMPRESS_COMMAND) { - Expression expressionTwo = getSecondExpr(); - checkNumParameters(getSecondExpr() != null ? 2 : 1); - checkMatrixFrameParam(getFirstExpr()); - if(expressionTwo != null) - checkMatrixParam(getSecondExpr()); + break; + } + case REMOVE: { + checkNumParameters(2); + checkListParam(getFirstExpr()); - Identifier compressInput1 = getFirstExpr().getOutput(); - // Identifier compressInput2 = getSecondExpr().getOutput(); + // setup output properties + DataIdentifier out1 = (DataIdentifier) getOutputs()[0]; + DataIdentifier out2 = (DataIdentifier) getOutputs()[1]; - DataIdentifier compressOutput = (DataIdentifier) getOutputs()[0]; - compressOutput.setDataType(DataType.MATRIX); - compressOutput.setDimensions(compressInput1.getDim1(), compressInput1.getDim2()); - compressOutput.setBlocksize(compressInput1.getBlocksize()); - compressOutput.setValueType(compressInput1.getValueType()); + // Output1 - list after removal + long nrow = getFirstExpr().getOutput().getDim1() > 0 ? getFirstExpr().getOutput().getDim1() + 1 : -1; + out1.setDataType(DataType.LIST); + out1.setValueType(getFirstExpr().getOutput().getValueType()); + out1.setDimensions(nrow, 1); + out1.setBlocksize(getFirstExpr().getOutput().getBlocksize()); - DataIdentifier metaOutput = (DataIdentifier) getOutputs()[1]; - metaOutput.setDataType(DataType.FRAME); - metaOutput.setDimensions(compressInput1.getDim1(), -1); + // Output2 - list of removed element + out2.setDataType(DataType.LIST); + out2.setValueType(getFirstExpr().getOutput().getValueType()); + out2.setDimensions(1, 1); + out2.setBlocksize(getFirstExpr().getOutput().getBlocksize()); + + break; } - else - raiseValidateError("Compress/DeCompress instruction not allowed in dml script"); - break; - case EINSUM: - validateEinsum((DataIdentifier) getOutputs()[0]); - break; - default: //always unconditional - raiseValidateError("Unknown Builtin Function opcode: " + _opcode, false); + case SVD: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + + long minMN = Math.min(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2()); + + // setup output properties + DataIdentifier svdOut1 = (DataIdentifier) getOutputs()[0]; + DataIdentifier svdOut2 = (DataIdentifier) getOutputs()[1]; + DataIdentifier svdOut3 = (DataIdentifier) getOutputs()[2]; + + // Output 1 + svdOut1.setDataType(DataType.MATRIX); + svdOut1.setValueType(ValueType.FP64); + svdOut1.setDimensions(getFirstExpr().getOutput().getDim1(), minMN); + svdOut1.setBlocksize(getFirstExpr().getOutput().getBlocksize()); + + // Output 2 + svdOut2.setDataType(DataType.MATRIX); + svdOut2.setValueType(ValueType.FP64); + svdOut2.setDimensions(minMN, minMN); + svdOut2.setBlocksize(getFirstExpr().getOutput().getBlocksize()); + + // Output 3 + svdOut3.setDataType(DataType.MATRIX); + svdOut3.setValueType(ValueType.FP64); + svdOut3.setDimensions(getFirstExpr().getOutput().getDim2(), minMN); + svdOut3.setBlocksize(getFirstExpr().getOutput().getBlocksize()); + + break; + + case COMPRESS: + if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_COMPRESS_COMMAND) { + Expression expressionTwo = getSecondExpr(); + checkNumParameters(getSecondExpr() != null ? 2 : 1); + checkMatrixFrameParam(getFirstExpr()); + if(expressionTwo != null) + checkMatrixParam(getSecondExpr()); + + Identifier compressInput1 = getFirstExpr().getOutput(); + // Identifier compressInput2 = getSecondExpr().getOutput(); + + DataIdentifier compressOutput = (DataIdentifier) getOutputs()[0]; + compressOutput.setDataType(DataType.MATRIX); + compressOutput.setDimensions(compressInput1.getDim1(), compressInput1.getDim2()); + compressOutput.setBlocksize(compressInput1.getBlocksize()); + compressOutput.setValueType(compressInput1.getValueType()); + + DataIdentifier metaOutput = (DataIdentifier) getOutputs()[1]; + metaOutput.setDataType(DataType.FRAME); + metaOutput.setDimensions(compressInput1.getDim1(), -1); + } + else + raiseValidateError("Compress/DeCompress instruction not allowed in dml script"); + break; + case EINSUM: + validateEinsum((DataIdentifier) getOutputs()[0]); + break; + default: //always unconditional + raiseValidateError("Unknown Builtin Function opcode: " + _opcode, false); } } private static boolean isPowerOfTwo(long n) { return (n > 0) && ((n & (n - 1)) == 0); } - + private static void setDimensions(DataIdentifier out, Expression exp) { out.setDataType(DataType.MATRIX); out.setValueType(ValueType.FP64); out.setDimensions(exp.getOutput().getDim1(), exp.getOutput().getDim2()); out.setBlocksize(exp.getOutput().getBlocksize()); } - - private static ArrayList orderDnnParams(ArrayList paramExpression, int skip) { + + private static ArrayList orderDnnParams(ArrayList paramExpression, + int skip) { ArrayList newParams = new ArrayList<>(); for(int i = 0; i < skip; i++) newParams.add(paramExpression.get(i)); - String [] orderedParams = { - "stride1", "stride2", "padding1", "padding2", - "input_shape1", "input_shape2", "input_shape3", "input_shape4", - "filter_shape1", "filter_shape2", "filter_shape3", "filter_shape4" - }; + String[] orderedParams = {"stride1", "stride2", "padding1", "padding2", "input_shape1", "input_shape2", + "input_shape3", "input_shape4", "filter_shape1", "filter_shape2", "filter_shape3", "filter_shape4"}; for(int i = 0; i < orderedParams.length; i++) { boolean found = false; for(ParameterExpression param : paramExpression) { - if(param.getName() != null && param.getName().equals(orderedParams[i])) { + if(param.getName() != null && param.getName().equals(orderedParams[i])) { found = true; newParams.add(param); } @@ -800,14 +797,15 @@ private static ArrayList orderDnnParams(ArrayList replaceListParams(ArrayList paramExpression, - String inputVarName, String outputVarName, int startIndex) { + String inputVarName, String outputVarName, int startIndex) { ArrayList newParamExpression = new ArrayList<>(); int i = startIndex; int j = 1; // Assumption: sequential ordering pool_size1, pool_size2 - for (ParameterExpression expr : paramExpression) { + for(ParameterExpression expr : paramExpression) { if(expr.getName() != null && expr.getName().equals(inputVarName + j)) { newParamExpression.add(new ParameterExpression(outputVarName + i, expr.getExpr())); - i++; j++; + i++; + j++; } else { newParamExpression.add(expr); @@ -816,21 +814,22 @@ private static ArrayList replaceListParams(ArrayList expandListParams(ArrayList paramExpression, - HashSet paramsToExpand) { + private static ArrayList expandListParams(ArrayList paramExpression, + HashSet paramsToExpand) { ArrayList newParamExpressions = new ArrayList<>(); for(ParameterExpression expr : paramExpression) { if(paramsToExpand.contains(expr.getName())) { if(expr.getExpr() instanceof ExpressionList) { int i = 1; - for(Expression e : ((ExpressionList)expr.getExpr()).getValue()) { + for(Expression e : ((ExpressionList) expr.getExpr()).getValue()) { newParamExpressions.add(new ParameterExpression(expr.getName() + i, e)); i++; } } } else if(expr.getExpr() instanceof ExpressionList) { - throw new LanguageException("The parameter " + expr.getName() + " cannot be list or is not supported for the given function"); + throw new LanguageException( + "The parameter " + expr.getName() + " cannot be list or is not supported for the given function"); } else { newParamExpressions.add(expr); @@ -838,20 +837,26 @@ else if(expr.getExpr() instanceof ExpressionList) { } return newParamExpressions; } - + private ArrayList expandDnnArguments(ArrayList paramExpression) { try { - if(_opcode == Builtins.CONV2D || _opcode == Builtins.CONV2D_BACKWARD_FILTER - || _opcode == Builtins.CONV2D_BACKWARD_DATA) { + if(_opcode == Builtins.CONV2D || _opcode == Builtins.CONV2D_BACKWARD_FILTER || + _opcode == Builtins.CONV2D_BACKWARD_DATA) { HashSet expand = new HashSet<>(); - expand.add("input_shape"); expand.add("filter_shape"); expand.add("stride"); expand.add("padding"); + expand.add("input_shape"); + expand.add("filter_shape"); + expand.add("stride"); + expand.add("padding"); paramExpression = expandListParams(paramExpression, expand); paramExpression = orderDnnParams(paramExpression, 2); } - else if(_opcode == Builtins.MAX_POOL || _opcode == Builtins.AVG_POOL || - _opcode == Builtins.MAX_POOL_BACKWARD || _opcode == Builtins.AVG_POOL_BACKWARD) { + else if(_opcode == Builtins.MAX_POOL || _opcode == Builtins.AVG_POOL || + _opcode == Builtins.MAX_POOL_BACKWARD || _opcode == Builtins.AVG_POOL_BACKWARD) { HashSet expand = new HashSet<>(); - expand.add("input_shape"); expand.add("pool_size"); expand.add("stride"); expand.add("padding"); + expand.add("input_shape"); + expand.add("pool_size"); + expand.add("stride"); + expand.add("padding"); paramExpression = expandListParams(paramExpression, expand); paramExpression.add(new ParameterExpression("filter_shape1", new IntIdentifier(1, this))); paramExpression.add(new ParameterExpression("filter_shape2", new IntIdentifier(1, this))); @@ -867,1284 +872,1331 @@ else if(_opcode == Builtins.MAX_POOL || _opcode == Builtins.AVG_POOL || } return paramExpression; } - + private boolean isValidNoArgumentFunction() { - return getOpCode() == Builtins.TIME - || getOpCode() == Builtins.LIST; + return getOpCode() == Builtins.TIME || getOpCode() == Builtins.LIST; } /** - * Validate parse tree : Process BuiltinFunction Expression in an assignment - * statement + * Validate parse tree : Process BuiltinFunction Expression in an assignment statement */ @Override - public void validateExpression(HashMap ids, HashMap constVars, boolean conditional) - { - for(int i=0; i < _args.length; i++ ) { - if (_args[i] instanceof FunctionCallIdentifier){ + public void validateExpression(HashMap ids, HashMap constVars, + boolean conditional) { + for(int i = 0; i < _args.length; i++) { + if(_args[i] instanceof FunctionCallIdentifier) { raiseValidateError("UDF function call not supported as parameter to built-in function call", false); } _args[i].validateExpression(ids, constVars, conditional); } - + // checkIdentifierParams(); String outputName = getTempName(); DataIdentifier output = new DataIdentifier(outputName); output.setParseInfo(this); - - if (getFirstExpr() == null && !isValidNoArgumentFunction()) { // time has no arguments + + if(getFirstExpr() == null && !isValidNoArgumentFunction()) { // time has no arguments raiseValidateError("Function " + this + " has no arguments.", false); } - Identifier id = (_args.length != 0) ? - getFirstExpr().getOutput() : null; - if (_args.length != 0) + Identifier id = (_args.length != 0) ? getFirstExpr().getOutput() : null; + if(_args.length != 0) output.setProperties(this.getFirstExpr().getOutput()); output.setNnz(-1); //conservatively, cannot use input nnz! setOutput(output); - - switch (getOpCode()) { - case EVAL: - case EVALLIST: - if (_args.length == 0) - raiseValidateError("Function eval should provide at least one argument, i.e., the function name.", false); - checkValueTypeParam(_args[0], ValueType.STRING); - boolean listReturn = (getOpCode()==Builtins.EVALLIST); - output.setDataType(listReturn ? DataType.LIST : DataType.MATRIX); - output.setValueType(listReturn ? ValueType.UNKNOWN : ValueType.FP64); - output.setDimensions(-1, -1); - output.setBlocksize(ConfigurationManager.getBlocksize()); - break; - case COLSUM: - case COLMAX: - case COLMIN: - case COLMEAN: - case COLPROD: - case COLSD: - case COLVAR: - // colSums(X); - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); - output.setDimensions(1, id.getDim2()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(id.getValueType()); - break; - case ROWSUM: - case ROWMAX: - case ROWINDEXMAX: - case ROWMIN: - case ROWINDEXMIN: - case ROWMEAN: - case ROWPROD: - case ROWSD: - case ROWVAR: - //rowSums(X); - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); - output.setDimensions(id.getDim1(), 1); - output.setBlocksize (id.getBlocksize()); - output.setValueType(id.getValueType()); - break; - case TRACE: - if(getFirstExpr().getOutput().dimsKnown() - && getFirstExpr().getOutput().getDim1() != getFirstExpr().getOutput().getDim2()) - { - raiseValidateError("Trace is only defined on squared matrices but found [" - +getFirstExpr().getOutput().getDim1()+"x"+getFirstExpr().getOutput().getDim2()+"].", conditional); - } - case SUM: - case PROD: - case SD: - case VAR: - // sum(X); - checkNumParameters(1); - checkMatrixTensorParam(getFirstExpr()); - output.setDataType(DataType.SCALAR); - output.setDimensions(0, 0); - output.setBlocksize(0); - switch (id.getValueType()) { - case INT64: - case INT32: - case UINT8: - case UINT4: - case BOOLEAN: - output.setValueType(ValueType.INT64); - break; - case STRING: - case CHARACTER: - case FP64: - case FP32: - case HASH32: - case HASH64: //default - output.setValueType(ValueType.FP64); - break; - case UNKNOWN: - throw new NotImplementedException(); - } - break; - - case MEAN: - //checkNumParameters(2, false); // mean(Y) or mean(Y,W) - if (getSecondExpr() != null) { - checkNumParameters(2); - } - else { + + switch(getOpCode()) { + case EVAL: + case EVALLIST: + if(_args.length == 0) + raiseValidateError("Function eval should provide at least one argument, i.e., the function name.", + false); + checkValueTypeParam(_args[0], ValueType.STRING); + boolean listReturn = (getOpCode() == Builtins.EVALLIST); + output.setDataType(listReturn ? DataType.LIST : DataType.MATRIX); + output.setValueType(listReturn ? ValueType.UNKNOWN : ValueType.FP64); + output.setDimensions(-1, -1); + output.setBlocksize(ConfigurationManager.getBlocksize()); + break; + case COLSUM: + case COLMAX: + case COLMIN: + case COLMEAN: + case COLPROD: + case COLSD: + case COLVAR: + // colSums(X); checkNumParameters(1); - } - - checkMatrixParam(getFirstExpr()); - if ( getSecondExpr() != null ) { - // x = mean(Y,W); - checkMatchingDimensions(getFirstExpr(), getSecondExpr()); - } - - output.setDataType(DataType.SCALAR); - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType(id.getValueType()); - break; - - case XOR: - case BITWAND: - case BITWOR: - case BITWXOR: - case BITWSHIFTL: - case BITWSHIFTR: - checkNumParameters(2); - setBinaryOutputProperties(output); - break; - - case MIN: - case MAX: - //min(X), min(X,s), min(s,X), min(s,r), min(X,Y) - if (getSecondExpr() == null) { //unary + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(1, id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + break; + case ROWSUM: + case ROWMAX: + case ROWINDEXMAX: + case ROWMIN: + case ROWINDEXMIN: + case ROWMEAN: + case ROWPROD: + case ROWSD: + case ROWVAR: + //rowSums(X); checkNumParameters(1); checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.SCALAR); + output.setDataType(DataType.MATRIX); + output.setDimensions(id.getDim1(), 1); + output.setBlocksize(id.getBlocksize()); output.setValueType(id.getValueType()); + break; + case TRACE: + if(getFirstExpr().getOutput().dimsKnown() && + getFirstExpr().getOutput().getDim1() != getFirstExpr().getOutput().getDim2()) { + raiseValidateError( + "Trace is only defined on squared matrices but found [" + getFirstExpr().getOutput().getDim1() + + "x" + getFirstExpr().getOutput().getDim2() + "].", conditional); + } + case SUM: + case PROD: + case SD: + case VAR: + // sum(X); + checkNumParameters(1); + checkMatrixTensorParam(getFirstExpr()); + output.setDataType(DataType.SCALAR); output.setDimensions(0, 0); output.setBlocksize(0); - } - else if( getAllExpr().length == 2 ) { //binary - checkNumParameters(2); - setBinaryOutputProperties(output); - } - else { //nary - for( Expression e : getAllExpr() ) - checkMatrixScalarParam(e); - setNaryOutputProperties(output); - } - break; - - case CUMSUM: - case ROWCUMSUM: - case CUMPROD: - case CUMSUMPROD: - case CUMMIN: - case CUMMAX: - // cumsum(X); - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - boolean cumSP = getOpCode() == Builtins.CUMSUMPROD; - if( cumSP && id.getDim2() > 2 ) - raiseValidateError("Cumsumprod only supported over two-column matrices", conditional); - - output.setDataType(DataType.MATRIX); - output.setDimensions(id.getDim1(), cumSP ? 1 : id.getDim2()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(id.getValueType()); - - break; - case CAST_AS_SCALAR: - checkNumParameters(1); - checkDataTypeParam(getFirstExpr(), - DataType.MATRIX, DataType.FRAME, DataType.LIST); - if (( getFirstExpr().getOutput().getDim1() != -1 && getFirstExpr().getOutput().getDim1() !=1) - || ( getFirstExpr().getOutput().getDim2() != -1 && getFirstExpr().getOutput().getDim2() !=1)) { - raiseValidateError("dimension mismatch while casting matrix to scalar: dim1: " + getFirstExpr().getOutput().getDim1() - + " dim2 " + getFirstExpr().getOutput().getDim2(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - output.setDataType(DataType.SCALAR); - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType((id.getValueType()!=ValueType.UNKNOWN - || id.getDataType()==DataType.LIST) ? id.getValueType() : ValueType.FP64); - break; - case CAST_AS_MATRIX: - checkNumParameters(1); - checkDataTypeParam(getFirstExpr(), - DataType.SCALAR, DataType.FRAME, DataType.LIST); - output.setDataType(DataType.MATRIX); - output.setDimensions(id.getDim1(), id.getDim2()); - if( getFirstExpr().getOutput().getDataType()==DataType.SCALAR ) - output.setDimensions(1, 1); //correction scalars - if( getFirstExpr().getOutput().getDataType()==DataType.LIST ) - output.setDimensions(-1, -1); //correction list: arbitrary object - output.setBlocksize(id.getBlocksize()); - output.setValueType(ValueType.FP64); //matrices always in double - break; - case CAST_AS_LIST: //list unnesting - checkNumParameters(1); - checkDataTypeParam(getFirstExpr(), DataType.LIST); - output.setDataType(DataType.LIST); - output.setDimensions(-1, 1); - output.setBlocksize(id.getBlocksize()); - output.setValueType(ValueType.UNKNOWN); - break; - case TYPEOF: - case DETECTSCHEMA: - case COLNAMES: - checkNumParameters(1); - checkMatrixFrameParam(getFirstExpr()); - output.setDataType(DataType.FRAME); - output.setDimensions(1, id.getDim2()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(ValueType.STRING); - break; - case CAST_AS_FRAME: - // operation as.frame - // overloaded to take either one argument or 2 where second is column names - if( getSecondExpr() == null) {// there is no column names - checkNumParameters(1); - } - else{ // there is column names - checkNumParameters(2); - checkDataTypeParam(getSecondExpr(), DataType.LIST); - } + switch(id.getValueType()) { + case INT64: + case INT32: + case UINT8: + case UINT4: + case BOOLEAN: + output.setValueType(ValueType.INT64); + break; + case STRING: + case CHARACTER: + case FP64: + case FP32: + case HASH32: + case HASH64: //default + output.setValueType(ValueType.FP64); + break; + case UNKNOWN: + throw new NotImplementedException(); + } + break; - checkDataTypeParam(getFirstExpr(), DataType.SCALAR, DataType.MATRIX, DataType.LIST); - output.setDataType(DataType.FRAME); - output.setDimensions(id.getDim1(), id.getDim2()); - if(getFirstExpr().getOutput().getDataType() == DataType.SCALAR) - output.setDimensions(1, 1); // correction scalars - if(getFirstExpr().getOutput().getDataType() == DataType.LIST) - output.setDimensions(-1, -1); // correction list: arbitrary object - output.setBlocksize(id.getBlocksize()); - output.setValueType(id.getValueType()); - break; - case CAST_AS_DOUBLE: - checkNumParameters(1); - checkScalarParam(getFirstExpr()); - output.setDataType(DataType.SCALAR); - //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType(ValueType.FP64); - break; - case CAST_AS_INT: - checkNumParameters(1); - checkScalarParam(getFirstExpr()); - output.setDataType(DataType.SCALAR); - //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType(ValueType.INT64); - break; - case CAST_AS_BOOLEAN: - checkNumParameters(1); - checkScalarParam(getFirstExpr()); - output.setDataType(DataType.SCALAR); - //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType(ValueType.BOOLEAN); - break; - - case IFELSE: - checkNumParameters(3); - setTernaryOutputProperties(output, conditional); - break; - - case CBIND: - case RBIND: - //scalar string append (string concatenation with \n) - if( getFirstExpr().getOutput().getDataType()==DataType.SCALAR ) { - checkNumParameters(2); - checkScalarParam(getFirstExpr()); - checkScalarParam(getSecondExpr()); - checkValueTypeParam(getFirstExpr(), ValueType.STRING); - checkValueTypeParam(getSecondExpr(), ValueType.STRING); - } - // append (rbind/cbind) all the elements of a list - else if( getAllExpr().length == 1 ) { - checkDataTypeParam(getFirstExpr(), DataType.LIST); - } - else { - if( getAllExpr().length < 2 ) - raiseValidateError("Invalid number of arguments for "+getOpCode(), conditional); - //list append - if(getFirstExpr().getOutput().getDataType().isList() ) - for(int i=1; i= 0 && m2rlen >= 0 && m1rlen!=m2rlen) { - raiseValidateError("inputs to cbind must have same number of rows: input 1 rows: " + - m1rlen+", input 2 rows: "+m2rlen, conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - appendDim1 = (m2rlen>=0) ? m2rlen : appendDim1; - appendDim2 = (appendDim2>=0 && m2clen>=0) ? appendDim2 + m2clen : -1; - } - else if( getOpCode() == Builtins.RBIND ) { - if (m1clen >= 0 && m2clen >= 0 && m1clen!=m2clen) { - raiseValidateError("inputs to rbind must have same number of columns: input 1 columns: " + - m1clen+", input 2 columns: "+m2clen, conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - appendDim1 = (appendDim1>=0 && m2rlen>=0)? appendDim1 + m2rlen : -1; - appendDim2 = (m2clen>=0) ? m2clen : appendDim2; - } + case MEAN: + //checkNumParameters(2, false); // mean(Y) or mean(Y,W) + if(getSecondExpr() != null) { + checkNumParameters(2); } - } - - output.setDimensions(appendDim1, appendDim2); - output.setBlocksize (id.getBlocksize()); - - break; - - case PPRED: - // TODO: remove this when ppred has been removed from DML - raiseValidateError("ppred() has been deprecated. Please use the operator directly.", true); - - // ppred (X,Y, "<"); ppred (X,y, "<"); ppred (y,X, "<"); - checkNumParameters(3); - - DataType dt1 = getFirstExpr().getOutput().getDataType(); - DataType dt2 = getSecondExpr().getOutput().getDataType(); - - //check input data types - if( dt1 == DataType.SCALAR && dt2 == DataType.SCALAR ) { - raiseValidateError("ppred() requires at least one matrix input.", conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - if( dt1 == DataType.MATRIX ) + else { + checkNumParameters(1); + } + checkMatrixParam(getFirstExpr()); - if( dt2 == DataType.MATRIX ) - checkMatrixParam(getSecondExpr()); - - //check operator - if (getThirdExpr().getOutput().getDataType() != DataType.SCALAR || - getThirdExpr().getOutput().getValueType() != ValueType.STRING) - { - raiseValidateError("Third argument in ppred() is not an operator ", conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - setBinaryOutputProperties(output); - break; - - case TRANS: - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); - output.setDimensions(id.getDim2(), id.getDim1()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(id.getValueType()); - break; - - case REV: - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); - output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(id.getValueType()); - break; - - case ROLL: - checkNumParameters(2); - checkMatrixParam(getFirstExpr()); - checkScalarParam(getSecondExpr()); - output.setDataType(DataType.MATRIX); - output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize(id.getBlocksize()); - output.setValueType(id.getValueType()); - break; - - case DIAG: - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); - if( id.getDim2() != -1 ) { //type known - if ( id.getDim2() == 1 ) - { - //diag V2M - output.setDimensions(id.getDim1(), id.getDim1()); - } - else - { - if (id.getDim1() != id.getDim2()) { - raiseValidateError("diag can either: (1) create diagonal matrix from (n x 1) matrix, or (2) take diagonal from a square matrix. " - + "Error invoking diag on matrix with dimensions (" - + id.getDim1() + "," + id.getDim2() - + ") in " + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - //diag M2V - output.setDimensions(id.getDim1(), 1); + if(getSecondExpr() != null) { + // x = mean(Y,W); + checkMatchingDimensions(getFirstExpr(), getSecondExpr()); } - } - output.setBlocksize(id.getBlocksize()); - output.setValueType(id.getValueType()); - break; - case DET: - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - if ( id.getDim2() == -1 || id.getDim1() != id.getDim2() ) { - raiseValidateError("det requires a square matrix as first argument.", conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - output.setDataType(DataType.SCALAR); - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType(ValueType.FP64); - break; - case NROW: - case NCOL: - case LENGTH: - checkNumParameters(1); - checkDataTypeParam(getFirstExpr(), - DataType.FRAME, DataType.LIST, DataType.MATRIX); - output.setDataType(DataType.SCALAR); - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType(ValueType.INT64); - break; - case LINEAGE: - checkNumParameters(1); - checkDataTypeParam(getFirstExpr(), - DataType.MATRIX, DataType.FRAME, DataType.LIST); - output.setDataType(DataType.SCALAR); - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType(ValueType.STRING); - break; - case LIST: - output.setDataType(DataType.LIST); - output.setValueType(ValueType.UNKNOWN); - output.setDimensions(getAllExpr().length, 1); - output.setBlocksize(-1); - break; - case EXISTS: - checkNumParameters(1); - checkStringOrDataIdentifier(getFirstExpr()); - output.setDataType(DataType.SCALAR); - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setValueType(ValueType.BOOLEAN); - break; - - // Contingency tables - case TABLE: - - /* - * Allowed #of arguments: 2,3,4,5,6 - * table(A,B) - * table(A,B,W) - * table(A,B,1) - * table(A,B,dim1,dim2) - * table(A,B,W,dim1,dim2) - * table(A,B,1,dim1,dim2) - * table(A,B,1,dim1,dim2,TRUE) - */ - - // Check for validity of input arguments, and setup output dimensions - - // First input: is always of type MATRIX - checkMatrixParam(getFirstExpr()); - - if (getSecondExpr() == null) - raiseValidateError("Invalid number of arguments to table(). " - + "The table() function requires 2, 3, 4, 5, or 6 arguments.", conditional); - - // Second input: can be MATRIX or SCALAR - // cases: table(A,B) or table(A,1) - if ( getSecondExpr().getOutput().getDataType() == DataType.MATRIX) - checkMatchingDimensions(getFirstExpr(),getSecondExpr()); - - long outputDim1=-1, outputDim2=-1; - - switch(_args.length) { - case 2: - // nothing to do - break; - - case 3: - // case - table w/ weights - // - weights specified as a matrix: table(A,B,W) or table(A,1,W) - // - weights specified as a scalar: table(A,B,1) or table(A,1,1) - if ( getThirdExpr().getOutput().getDataType() == DataType.MATRIX) - checkMatchingDimensions(getFirstExpr(),getThirdExpr()); - break; - - case 4: - // case - table w/ output dimensions: table(A,B,dim1,dim2) or table(A,1,dim1,dim2) - // third and fourth arguments must be scalars - if ( getThirdExpr().getOutput().getDataType() != DataType.SCALAR || _args[3].getOutput().getDataType() != DataType.SCALAR ) { - raiseValidateError("Invalid argument types to table(): output dimensions must be of type scalar: " - + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); + + output.setDataType(DataType.SCALAR); + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType(id.getValueType()); + break; + + case XOR: + case BITWAND: + case BITWOR: + case BITWXOR: + case BITWSHIFTL: + case BITWSHIFTR: + checkNumParameters(2); + setBinaryOutputProperties(output); + break; + + case MIN: + case MAX: + //min(X), min(X,s), min(s,X), min(s,r), min(X,Y) + if(getSecondExpr() == null) { //unary + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.SCALAR); + output.setValueType(id.getValueType()); + output.setDimensions(0, 0); + output.setBlocksize(0); + } + else if(getAllExpr().length == 2) { //binary + checkNumParameters(2); + setBinaryOutputProperties(output); + } + else { //nary + for(Expression e : getAllExpr()) + checkMatrixScalarParam(e); + setNaryOutputProperties(output); + } + break; + + case CUMSUM: + case ROWCUMSUM: + case CUMPROD: + case CUMSUMPROD: + case CUMMIN: + case CUMMAX: + // cumsum(X); + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + boolean cumSP = getOpCode() == Builtins.CUMSUMPROD; + if(cumSP && id.getDim2() > 2) + raiseValidateError("Cumsumprod only supported over two-column matrices", conditional); + + output.setDataType(DataType.MATRIX); + output.setDimensions(id.getDim1(), cumSP ? 1 : id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + + break; + case CAST_AS_SCALAR: + checkNumParameters(1); + checkDataTypeParam(getFirstExpr(), DataType.MATRIX, DataType.FRAME, DataType.LIST); + if((getFirstExpr().getOutput().getDim1() != -1 && getFirstExpr().getOutput().getDim1() != 1) || + (getFirstExpr().getOutput().getDim2() != -1 && getFirstExpr().getOutput().getDim2() != 1)) { + raiseValidateError("dimension mismatch while casting matrix to scalar: dim1: " + + getFirstExpr().getOutput().getDim1() + " dim2 " + getFirstExpr().getOutput().getDim2(), + conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + output.setDataType(DataType.SCALAR); + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType((id.getValueType() != ValueType.UNKNOWN || + id.getDataType() == DataType.LIST) ? id.getValueType() : ValueType.FP64); + break; + case CAST_AS_MATRIX: + checkNumParameters(1); + checkDataTypeParam(getFirstExpr(), DataType.SCALAR, DataType.FRAME, DataType.LIST); + output.setDataType(DataType.MATRIX); + output.setDimensions(id.getDim1(), id.getDim2()); + if(getFirstExpr().getOutput().getDataType() == DataType.SCALAR) + output.setDimensions(1, 1); //correction scalars + if(getFirstExpr().getOutput().getDataType() == DataType.LIST) + output.setDimensions(-1, -1); //correction list: arbitrary object + output.setBlocksize(id.getBlocksize()); + output.setValueType(ValueType.FP64); //matrices always in double + break; + case CAST_AS_LIST: //list unnesting + checkNumParameters(1); + checkDataTypeParam(getFirstExpr(), DataType.LIST); + output.setDataType(DataType.LIST); + output.setDimensions(-1, 1); + output.setBlocksize(id.getBlocksize()); + output.setValueType(ValueType.UNKNOWN); + break; + case TYPEOF: + case DETECTSCHEMA: + case COLNAMES: + case GET_NAMES: + checkNumParameters(1); + checkMatrixFrameParam(getFirstExpr()); + output.setDataType(DataType.FRAME); + output.setDimensions(1, id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(ValueType.STRING); + break; + case SET_NAMES: + //check if we use 2 parameters (Frame on which nemas are set and vector for names) + checkNumParameters(2); + + // check if first paramters is a frame + checkMatrixFrameParam(getFirstExpr()); + + // check if second paramters is a vector 1xn Frame + checkMatrixFrameParam(getSecondExpr()); + + //output should be a frame + output.setDataType(DataType.FRAME); + + checkMatrixFrameParam(getFirstExpr()); + output.setDataType(DataType.FRAME); + output.setDimensions(id.getDim1(), id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setDataType(DataType.FRAME); + break; + case CAST_AS_FRAME: + // operation as.frame + // overloaded to take either one argument or 2 where second is column names + if(getSecondExpr() == null) {// there is no column names + checkNumParameters(1); + } + else { // there is column names + checkNumParameters(2); + checkDataTypeParam(getSecondExpr(), DataType.LIST); + } + + checkDataTypeParam(getFirstExpr(), DataType.SCALAR, DataType.MATRIX, DataType.LIST); + output.setDataType(DataType.FRAME); + output.setDimensions(id.getDim1(), id.getDim2()); + if(getFirstExpr().getOutput().getDataType() == DataType.SCALAR) + output.setDimensions(1, 1); // correction scalars + if(getFirstExpr().getOutput().getDataType() == DataType.LIST) + output.setDimensions(-1, -1); // correction list: arbitrary object + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + break; + case CAST_AS_DOUBLE: + checkNumParameters(1); + checkScalarParam(getFirstExpr()); + output.setDataType(DataType.SCALAR); + //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType(ValueType.FP64); + break; + case CAST_AS_INT: + checkNumParameters(1); + checkScalarParam(getFirstExpr()); + output.setDataType(DataType.SCALAR); + //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType(ValueType.INT64); + break; + case CAST_AS_BOOLEAN: + checkNumParameters(1); + checkScalarParam(getFirstExpr()); + output.setDataType(DataType.SCALAR); + //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType(ValueType.BOOLEAN); + break; + + case IFELSE: + checkNumParameters(3); + setTernaryOutputProperties(output, conditional); + break; + + case CBIND: + case RBIND: + //scalar string append (string concatenation with \n) + if(getFirstExpr().getOutput().getDataType() == DataType.SCALAR) { + checkNumParameters(2); + checkScalarParam(getFirstExpr()); + checkScalarParam(getSecondExpr()); + checkValueTypeParam(getFirstExpr(), ValueType.STRING); + checkValueTypeParam(getSecondExpr(), ValueType.STRING); + } + // append (rbind/cbind) all the elements of a list + else if(getAllExpr().length == 1) { + checkDataTypeParam(getFirstExpr(), DataType.LIST); } else { - // constant propagation - if( getThirdExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getThirdExpr()).getName()) && !conditional ) - _args[2] = constVars.get(((DataIdentifier)getThirdExpr()).getName()); - if( _args[3] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[3]).getName()) && !conditional ) - _args[3] = constVars.get(((DataIdentifier)_args[3]).getName()); - - if ( getThirdExpr().getOutput() instanceof ConstIdentifier ) - outputDim1 = ((ConstIdentifier) getThirdExpr().getOutput()).getLongValue(); - if ( _args[3].getOutput() instanceof ConstIdentifier ) - outputDim2 = ((ConstIdentifier) _args[3].getOutput()).getLongValue(); - } - break; - - case 5: - case 6: - // case - table w/ weights and output dimensions: - // - table(A,B,W,dim1,dim2) or table(A,1,W,dim1,dim2) - // - table(A,B,1,dim1,dim2) or table(A,1,1,dim1,dim2) - - if ( getThirdExpr().getOutput().getDataType() == DataType.MATRIX) - checkMatchingDimensions(getFirstExpr(),getThirdExpr()); - - // fourth and fifth arguments must be scalars - if ( _args[3].getOutput().getDataType() != DataType.SCALAR || _args[4].getOutput().getDataType() != DataType.SCALAR ) { - raiseValidateError("Invalid argument types to table(): output dimensions must be of type scalar: " - + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); + if(getAllExpr().length < 2) + raiseValidateError("Invalid number of arguments for " + getOpCode(), conditional); + //list append + if(getFirstExpr().getOutput().getDataType().isList()) + for(int i = 1; i < getAllExpr().length; i++) + checkDataTypeParam(getExpr(i), DataType.SCALAR, DataType.MATRIX, DataType.FRAME, + DataType.LIST); + //matrix append (rbind/cbind) + else + for(int i = 0; i < getAllExpr().length; i++) + checkMatrixFrameParam(getExpr(i)); + } + + output.setDataType(id.getDataType()); + output.setValueType(id.getValueType()); + + //special handling of concatenating all list elements + if(id.getDataType() == DataType.LIST && getAllExpr().length == 1) { + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); + } + + // set output dimensions and validate consistency + long m1rlen = getFirstExpr().getOutput().getDim1(); + long m1clen = getFirstExpr().getOutput().getDim2(); + long appendDim1 = m1rlen, appendDim2 = m1clen; + + // best-effort dimension propagation and validation + if(id.getDataType() == DataType.LIST) { + appendDim1 = -1; + appendDim2 = -1; } else { - // constant propagation - if( _args[3] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[3]).getName()) && !conditional ) - _args[3] = constVars.get(((DataIdentifier)_args[3]).getName()); - if( _args[4] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[4]).getName()) && !conditional ) - _args[4] = constVars.get(((DataIdentifier)_args[4]).getName()); - - if ( _args[3].getOutput() instanceof ConstIdentifier ) - outputDim1 = ((ConstIdentifier) _args[3].getOutput()).getLongValue(); - if ( _args[4].getOutput() instanceof ConstIdentifier ) - outputDim2 = ((ConstIdentifier) _args[4].getOutput()).getLongValue(); + for(int i = 1; i < getAllExpr().length; i++) { + long m2rlen = getExpr(i).getOutput().getDim1(); + long m2clen = getExpr(i).getOutput().getDim2(); + + if(getOpCode() == Builtins.CBIND) { + if(m1rlen >= 0 && m2rlen >= 0 && m1rlen != m2rlen) { + raiseValidateError( + "inputs to cbind must have same number of rows: input 1 rows: " + m1rlen + + ", input 2 rows: " + m2rlen, conditional, + LanguageErrorCodes.INVALID_PARAMETERS); + } + appendDim1 = (m2rlen >= 0) ? m2rlen : appendDim1; + appendDim2 = (appendDim2 >= 0 && m2clen >= 0) ? appendDim2 + m2clen : -1; + } + else if(getOpCode() == Builtins.RBIND) { + if(m1clen >= 0 && m2clen >= 0 && m1clen != m2clen) { + raiseValidateError( + "inputs to rbind must have same number of columns: input 1 columns: " + m1clen + + ", input 2 columns: " + m2clen, conditional, + LanguageErrorCodes.INVALID_PARAMETERS); + } + appendDim1 = (appendDim1 >= 0 && m2rlen >= 0) ? appendDim1 + m2rlen : -1; + appendDim2 = (m2clen >= 0) ? m2clen : appendDim2; + } + } + } + + output.setDimensions(appendDim1, appendDim2); + output.setBlocksize(id.getBlocksize()); + + break; + + case PPRED: + // TODO: remove this when ppred has been removed from DML + raiseValidateError("ppred() has been deprecated. Please use the operator directly.", true); + + // ppred (X,Y, "<"); ppred (X,y, "<"); ppred (y,X, "<"); + checkNumParameters(3); + + DataType dt1 = getFirstExpr().getOutput().getDataType(); + DataType dt2 = getSecondExpr().getOutput().getDataType(); + + //check input data types + if(dt1 == DataType.SCALAR && dt2 == DataType.SCALAR) { + raiseValidateError("ppred() requires at least one matrix input.", conditional, + LanguageErrorCodes.INVALID_PARAMETERS); + } + if(dt1 == DataType.MATRIX) + checkMatrixParam(getFirstExpr()); + if(dt2 == DataType.MATRIX) + checkMatrixParam(getSecondExpr()); + + //check operator + if(getThirdExpr().getOutput().getDataType() != DataType.SCALAR || + getThirdExpr().getOutput().getValueType() != ValueType.STRING) { + raiseValidateError("Third argument in ppred() is not an operator ", conditional, + LanguageErrorCodes.INVALID_PARAMETERS); } - if( _args.length == 6 ) { - if( !_args[5].getOutput().isScalarBoolean() ) - raiseValidateError("The 6th ctable parameter (outputEmptyBlocks) must be a boolean literal.", conditional); + + setBinaryOutputProperties(output); + break; + + case TRANS: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(id.getDim2(), id.getDim1()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + break; + + case REV: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(id.getDim1(), id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + break; + + case ROLL: + checkNumParameters(2); + checkMatrixParam(getFirstExpr()); + checkScalarParam(getSecondExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(id.getDim1(), id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + break; + + case DIAG: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.MATRIX); + if(id.getDim2() != -1) { //type known + if(id.getDim2() == 1) { + //diag V2M + output.setDimensions(id.getDim1(), id.getDim1()); + } + else { + if(id.getDim1() != id.getDim2()) { + raiseValidateError( + "diag can either: (1) create diagonal matrix from (n x 1) matrix, or (2) take diagonal from a square matrix. " + + "Error invoking diag on matrix with dimensions (" + id.getDim1() + "," + + id.getDim2() + ") in " + this.toString(), conditional, + LanguageErrorCodes.INVALID_PARAMETERS); + } + //diag M2V + output.setDimensions(id.getDim1(), 1); + } + } + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + break; + case DET: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + if(id.getDim2() == -1 || id.getDim1() != id.getDim2()) { + raiseValidateError("det requires a square matrix as first argument.", conditional, + LanguageErrorCodes.INVALID_PARAMETERS); + } + output.setDataType(DataType.SCALAR); + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType(ValueType.FP64); + break; + case NROW: + case NCOL: + case LENGTH: + checkNumParameters(1); + checkDataTypeParam(getFirstExpr(), DataType.FRAME, DataType.LIST, DataType.MATRIX); + output.setDataType(DataType.SCALAR); + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType(ValueType.INT64); + break; + case LINEAGE: + checkNumParameters(1); + checkDataTypeParam(getFirstExpr(), DataType.MATRIX, DataType.FRAME, DataType.LIST); + output.setDataType(DataType.SCALAR); + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType(ValueType.STRING); + break; + case LIST: + output.setDataType(DataType.LIST); + output.setValueType(ValueType.UNKNOWN); + output.setDimensions(getAllExpr().length, 1); + output.setBlocksize(-1); + break; + case EXISTS: + checkNumParameters(1); + checkStringOrDataIdentifier(getFirstExpr()); + output.setDataType(DataType.SCALAR); + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setValueType(ValueType.BOOLEAN); + break; + + // Contingency tables + case TABLE: + + /* + * Allowed #of arguments: 2,3,4,5,6 + * table(A,B) + * table(A,B,W) + * table(A,B,1) + * table(A,B,dim1,dim2) + * table(A,B,W,dim1,dim2) + * table(A,B,1,dim1,dim2) + * table(A,B,1,dim1,dim2,TRUE) + */ + + // Check for validity of input arguments, and setup output dimensions + + // First input: is always of type MATRIX + checkMatrixParam(getFirstExpr()); + + if(getSecondExpr() == null) + raiseValidateError("Invalid number of arguments to table(). " + + "The table() function requires 2, 3, 4, 5, or 6 arguments.", conditional); + + // Second input: can be MATRIX or SCALAR + // cases: table(A,B) or table(A,1) + if(getSecondExpr().getOutput().getDataType() == DataType.MATRIX) + checkMatchingDimensions(getFirstExpr(), getSecondExpr()); + + long outputDim1 = -1, outputDim2 = -1; + + switch(_args.length) { + case 2: + // nothing to do + break; + + case 3: + // case - table w/ weights + // - weights specified as a matrix: table(A,B,W) or table(A,1,W) + // - weights specified as a scalar: table(A,B,1) or table(A,1,1) + if(getThirdExpr().getOutput().getDataType() == DataType.MATRIX) + checkMatchingDimensions(getFirstExpr(), getThirdExpr()); + break; + + case 4: + // case - table w/ output dimensions: table(A,B,dim1,dim2) or table(A,1,dim1,dim2) + // third and fourth arguments must be scalars + if(getThirdExpr().getOutput().getDataType() != DataType.SCALAR || + _args[3].getOutput().getDataType() != DataType.SCALAR) { + raiseValidateError( + "Invalid argument types to table(): output dimensions must be of type scalar: " + + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + else { + // constant propagation + if(getThirdExpr() instanceof DataIdentifier && + constVars.containsKey(((DataIdentifier) getThirdExpr()).getName()) && !conditional) + _args[2] = constVars.get(((DataIdentifier) getThirdExpr()).getName()); + if(_args[3] instanceof DataIdentifier && + constVars.containsKey(((DataIdentifier) _args[3]).getName()) && !conditional) + _args[3] = constVars.get(((DataIdentifier) _args[3]).getName()); + + if(getThirdExpr().getOutput() instanceof ConstIdentifier) + outputDim1 = ((ConstIdentifier) getThirdExpr().getOutput()).getLongValue(); + if(_args[3].getOutput() instanceof ConstIdentifier) + outputDim2 = ((ConstIdentifier) _args[3].getOutput()).getLongValue(); + } + break; + + case 5: + case 6: + // case - table w/ weights and output dimensions: + // - table(A,B,W,dim1,dim2) or table(A,1,W,dim1,dim2) + // - table(A,B,1,dim1,dim2) or table(A,1,1,dim1,dim2) + + if(getThirdExpr().getOutput().getDataType() == DataType.MATRIX) + checkMatchingDimensions(getFirstExpr(), getThirdExpr()); + + // fourth and fifth arguments must be scalars + if(_args[3].getOutput().getDataType() != DataType.SCALAR || + _args[4].getOutput().getDataType() != DataType.SCALAR) { + raiseValidateError( + "Invalid argument types to table(): output dimensions must be of type scalar: " + + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + else { + // constant propagation + if(_args[3] instanceof DataIdentifier && + constVars.containsKey(((DataIdentifier) _args[3]).getName()) && !conditional) + _args[3] = constVars.get(((DataIdentifier) _args[3]).getName()); + if(_args[4] instanceof DataIdentifier && + constVars.containsKey(((DataIdentifier) _args[4]).getName()) && !conditional) + _args[4] = constVars.get(((DataIdentifier) _args[4]).getName()); + + if(_args[3].getOutput() instanceof ConstIdentifier) + outputDim1 = ((ConstIdentifier) _args[3].getOutput()).getLongValue(); + if(_args[4].getOutput() instanceof ConstIdentifier) + outputDim2 = ((ConstIdentifier) _args[4].getOutput()).getLongValue(); + } + if(_args.length == 6) { + if(!_args[5].getOutput().isScalarBoolean()) + raiseValidateError( + "The 6th ctable parameter (outputEmptyBlocks) must be a boolean literal.", + conditional); + } + break; + + default: + raiseValidateError("Invalid number of arguments to table(): " + this.toString(), conditional, + LanguageErrorCodes.INVALID_PARAMETERS); + } + // The dimensions for the output matrix will be known only at the + // run time + output.setDimensions(outputDim1, outputDim2); + output.setBlocksize(-1); + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); + break; + + case MOMENT: + checkMatrixParam(getFirstExpr()); + if(getThirdExpr() != null) { + checkNumParameters(3); + checkMatrixParam(getSecondExpr()); + checkMatchingDimensions(getFirstExpr(), getSecondExpr()); + checkScalarParam(getThirdExpr()); + } + else { + checkNumParameters(2); + checkScalarParam(getSecondExpr()); + } + + // output is a scalar + output.setDataType(DataType.SCALAR); + output.setValueType(ValueType.FP64); + output.setDimensions(0, 0); + output.setBlocksize(0); + break; + + case COV: + /* + * x = cov(V1,V2) or xw = cov(V1,V2,W) + */ + if(getThirdExpr() != null) { + checkNumParameters(3); + } + else { + checkNumParameters(2); + } + checkMatrixParam(getFirstExpr()); + checkMatrixParam(getSecondExpr()); + checkMatchingDimensions(getFirstExpr(), getSecondExpr()); + + if(getThirdExpr() != null) { + checkMatrixParam(getThirdExpr()); + checkMatchingDimensions(getFirstExpr(), getThirdExpr()); + } + + // output is a scalar + output.setDataType(DataType.SCALAR); + output.setValueType(ValueType.FP64); + output.setDimensions(0, 0); + output.setBlocksize(0); + break; + + case QUANTILE: + /* + * q = quantile(V1,0.5) computes median in V1 + * or Q = quantile(V1,P) computes the vector of quantiles as specified by P + * or qw = quantile(V1,W,0.5) computes median when weights (W) are given + * or QW = quantile(V1,W,P) computes the vector of quantiles as specified by P, when weights (W) are given + */ + if(getThirdExpr() != null) { + checkNumParameters(3); + } + else { + checkNumParameters(2); + } + + // first parameter must always be a 1D matrix + check1DMatrixParam(getFirstExpr()); + + // check for matching dimensions for other matrix parameters + if(getThirdExpr() != null) { + checkMatrixParam(getSecondExpr()); + checkMatchingDimensions(getFirstExpr(), getSecondExpr()); + } + + // set the properties for _output expression + // output dimensions = dimensions of second, if third is null + // = dimensions of the third, otherwise. + + if(getThirdExpr() != null) { + output.setDimensions(getThirdExpr().getOutput().getDim1(), getThirdExpr().getOutput().getDim2()); + output.setBlocksize(getThirdExpr().getOutput().getBlocksize()); + output.setDataType(getThirdExpr().getOutput().getDataType()); + } + else { + output.setDimensions(getSecondExpr().getOutput().getDim1(), getSecondExpr().getOutput().getDim2()); + output.setBlocksize(getSecondExpr().getOutput().getBlocksize()); + output.setDataType(getSecondExpr().getOutput().getDataType()); + } + break; + + case INTERQUANTILE: + if(getThirdExpr() != null) { + checkNumParameters(3); + } + else { + checkNumParameters(2); + } + checkMatrixParam(getFirstExpr()); + if(getThirdExpr() != null) { + // i.e., second input is weight vector + checkMatrixParam(getSecondExpr()); + checkMatchingDimensionsQuantile(); + } + + if((getThirdExpr() == null && getSecondExpr().getOutput().getDataType() != DataType.SCALAR) && + (getThirdExpr() != null && getThirdExpr().getOutput().getDataType() != DataType.SCALAR)) { + + raiseValidateError("Invalid parameters to " + this.getOpCode(), conditional, + LanguageErrorCodes.INVALID_PARAMETERS); + } + + output.setValueType(id.getValueType()); + // output dimensions are unknown + output.setDimensions(-1, -1); + output.setBlocksize(-1); + output.setDataType(DataType.MATRIX); + break; + + case IQM: + /* + * Usage: iqm = InterQuartileMean(A,W); iqm = InterQuartileMean(A); + */ + if(getSecondExpr() != null) { + checkNumParameters(2); + } + else { + checkNumParameters(1); + } + checkMatrixParam(getFirstExpr()); + + if(getSecondExpr() != null) { + // i.e., second input is weight vector + checkMatrixParam(getSecondExpr()); + checkMatchingDimensions(getFirstExpr(), getSecondExpr()); + } + + // Output is a scalar + output.setValueType(id.getValueType()); + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setDataType(DataType.SCALAR); + + break; + + case ISNA: + case ISNAN: + case ISINF: + checkNumParameters(1); + checkMatrixScalarParam(getFirstExpr()); + output.setDataType(id.getDataType()); + output.setDimensions(id.getDim1(), id.getDim2()); + output.setBlocksize(id.getBlocksize()); + //TODO set output type to boolean when supported + output.setValueType(id.getValueType()); + break; + + case MEDIAN: + checkNumParameters((getSecondExpr() != null) ? 2 : 1); + checkMatrixParam(getFirstExpr()); + + if(getSecondExpr() != null) { + // i.e., second input is weight vector + checkMatrixParam(getSecondExpr()); + checkMatchingDimensions(getFirstExpr(), getSecondExpr()); + } + + // Output is a scalar + output.setValueType(id.getValueType()); + output.setDimensions(0, 0); + output.setBlocksize(0); + output.setDataType(DataType.SCALAR); + + break; + + case SAMPLE: { + Expression[] in = getAllExpr(); + + for(Expression e : in) + checkScalarParam(e); + + if(in[0].getOutput().getValueType() != ValueType.FP64 && + in[0].getOutput().getValueType() != ValueType.INT64) + throw new LanguageException("First argument to sample() must be a number."); + if(in[1].getOutput().getValueType() != ValueType.FP64 && + in[1].getOutput().getValueType() != ValueType.INT64) + throw new LanguageException("Second argument to sample() must be a number."); + + boolean check = false; + if(isConstant(in[0]) && isConstant(in[1])) { + long range = ((ConstIdentifier) in[0]).getLongValue(); + long size = ((ConstIdentifier) in[1]).getLongValue(); + if(range < size) + check = true; + } + + if(in.length == 4) { + checkNumParameters(4); + if(in[3].getOutput().getValueType() != ValueType.INT64) + throw new LanguageException("Fourth argument, seed, to sample() must be an integer value."); + if(in[2].getOutput().getValueType() != ValueType.BOOLEAN) + throw new LanguageException( + "Third argument to sample() must either denote replacement policy (boolean) or seed (integer)."); + } + else if(in.length == 3) { + checkNumParameters(3); + if(in[2].getOutput().getValueType() != ValueType.BOOLEAN && + in[2].getOutput().getValueType() != ValueType.INT64) + throw new LanguageException( + "Third argument to sample() must either denote replacement policy (boolean) or seed (integer)."); + } + + if(check && in.length >= 3 && isConstant(in[2]) && + in[2].getOutput().getValueType() == ValueType.BOOLEAN && !((BooleanIdentifier) in[2]).getValue()) + throw new LanguageException( + "Sample (size=" + ((ConstIdentifier) in[0]).getLongValue() + ") larger than population (size=" + + ((ConstIdentifier) in[1]).getLongValue() + ") can only be generated with replacement."); + + // Output is a column vector + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); + + if(isConstant(in[1])) + output.setDimensions(((ConstIdentifier) in[1]).getLongValue(), 1); + else + output.setDimensions(-1, 1); + setBlocksize(id.getBlocksize()); + + break; + } + case SEQ: + + //basic parameter validation + checkScalarParam(getFirstExpr()); + checkScalarParam(getSecondExpr()); + if(getThirdExpr() != null) { + checkNumParameters(3); + checkScalarParam(getThirdExpr()); + } + else + checkNumParameters(2); + + // constant propagation (from, to, incr) + if(!conditional) { + if(getFirstExpr() instanceof DataIdentifier && + constVars.containsKey(((DataIdentifier) getFirstExpr()).getName())) + _args[0] = constVars.get(((DataIdentifier) getFirstExpr()).getName()); + if(getSecondExpr() instanceof DataIdentifier && + constVars.containsKey(((DataIdentifier) getSecondExpr()).getName())) + _args[1] = constVars.get(((DataIdentifier) getSecondExpr()).getName()); + if(getThirdExpr() != null && getThirdExpr() instanceof DataIdentifier && + constVars.containsKey(((DataIdentifier) getThirdExpr()).getName())) + _args[2] = constVars.get(((DataIdentifier) getThirdExpr()).getName()); + } + + // check if dimensions can be inferred + long dim1 = -1, dim2 = 1; + if(isConstant(getFirstExpr()) && isConstant(getSecondExpr()) && + (getThirdExpr() != null ? isConstant(getThirdExpr()) : true)) { + double from, to, incr; + try { + from = getDoubleValue(getFirstExpr()); + to = getDoubleValue(getSecondExpr()); + + // Setup the value of increment + // default value: 1 if from <= to; -1 if from > to + if(getThirdExpr() == null) { + expandArguments(); + _args[2] = new DoubleIdentifier(((from > to) ? -1.0 : 1.0), this); + } + incr = getDoubleValue(getThirdExpr()); + + } + catch(LanguageException e) { + throw new LanguageException("Arguments for seq() must be numeric."); + } + + if((from > to) && (incr >= 0)) + throw new LanguageException("Wrong sign for the increment in a call to seq()"); + + // Both end points of the range must included i.e., [from,to] both inclusive. + // Note that, "to" is included only if (to-from) is perfectly divisible by incr + // For example, seq(0,1,0.5) produces (0.0 0.5 1.0) whereas seq(0,1,0.6) produces only (0.0 0.6) but not (0.0 0.6 1.0) + dim1 = UtilFunctions.getSeqLength(from, to, incr); } + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); + output.setDimensions(dim1, dim2); + output.setBlocksize(0); break; - default: - raiseValidateError("Invalid number of arguments to table(): " - + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - // The dimensions for the output matrix will be known only at the - // run time - output.setDimensions(outputDim1, outputDim2); - output.setBlocksize (-1); - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - break; - - case MOMENT: - checkMatrixParam(getFirstExpr()); - if (getThirdExpr() != null) { - checkNumParameters(3); - checkMatrixParam(getSecondExpr()); - checkMatchingDimensions(getFirstExpr(),getSecondExpr()); - checkScalarParam(getThirdExpr()); - } - else { + case SOLVE: checkNumParameters(2); - checkScalarParam(getSecondExpr()); - } + checkMatrixParam(getFirstExpr()); + checkMatrixParam(getSecondExpr()); - // output is a scalar - output.setDataType(DataType.SCALAR); - output.setValueType(ValueType.FP64); - output.setDimensions(0, 0); - output.setBlocksize(0); - break; - - case COV: - /* - * x = cov(V1,V2) or xw = cov(V1,V2,W) - */ - if (getThirdExpr() != null) { - checkNumParameters(3); - } - else { - checkNumParameters(2); - } - checkMatrixParam(getFirstExpr()); - checkMatrixParam(getSecondExpr()); - checkMatchingDimensions(getFirstExpr(),getSecondExpr()); - - if (getThirdExpr() != null) { - checkMatrixParam(getThirdExpr()); - checkMatchingDimensions(getFirstExpr(), getThirdExpr()); - } + if(getSecondExpr().getOutput().dimsKnown() && !is1DMatrix(getSecondExpr())) + raiseValidateError("Second input to solve() must be a vector", conditional); - // output is a scalar - output.setDataType(DataType.SCALAR); - output.setValueType(ValueType.FP64); - output.setDimensions(0, 0); - output.setBlocksize(0); - break; - - case QUANTILE: - /* - * q = quantile(V1,0.5) computes median in V1 - * or Q = quantile(V1,P) computes the vector of quantiles as specified by P - * or qw = quantile(V1,W,0.5) computes median when weights (W) are given - * or QW = quantile(V1,W,P) computes the vector of quantiles as specified by P, when weights (W) are given - */ - if(getThirdExpr() != null) { - checkNumParameters(3); - } - else { - checkNumParameters(2); - } - - // first parameter must always be a 1D matrix - check1DMatrixParam(getFirstExpr()); - - // check for matching dimensions for other matrix parameters - if (getThirdExpr() != null) { - checkMatrixParam(getSecondExpr()); - checkMatchingDimensions(getFirstExpr(), getSecondExpr()); - } - - // set the properties for _output expression - // output dimensions = dimensions of second, if third is null - // = dimensions of the third, otherwise. - - if (getThirdExpr() != null) { - output.setDimensions(getThirdExpr().getOutput().getDim1(), getThirdExpr().getOutput().getDim2()); - output.setBlocksize(getThirdExpr().getOutput().getBlocksize()); - output.setDataType(getThirdExpr().getOutput().getDataType()); - } else { - output.setDimensions(getSecondExpr().getOutput().getDim1(), getSecondExpr().getOutput().getDim2()); - output.setBlocksize(getSecondExpr().getOutput().getBlocksize()); - output.setDataType(getSecondExpr().getOutput().getDataType()); - } - break; + if(getFirstExpr().getOutput().dimsKnown() && getSecondExpr().getOutput().dimsKnown() && + getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1() && + getFirstExpr().getOutput().getDim1() != getFirstExpr().getOutput().getDim2()) + raiseValidateError("Dimension mismatch in a call to solve()", conditional); - case INTERQUANTILE: - if (getThirdExpr() != null) { - checkNumParameters(3); - } - else { - checkNumParameters(2); - } - checkMatrixParam(getFirstExpr()); - if (getThirdExpr() != null) { - // i.e., second input is weight vector - checkMatrixParam(getSecondExpr()); - checkMatchingDimensionsQuantile(); - } + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); + output.setDimensions(getFirstExpr().getOutput().getDim2(), 1); + output.setBlocksize(0); + break; - if ((getThirdExpr() == null && getSecondExpr().getOutput().getDataType() != DataType.SCALAR) - && (getThirdExpr() != null && getThirdExpr().getOutput().getDataType() != DataType.SCALAR)) { - - raiseValidateError("Invalid parameters to "+ this.getOpCode(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } + case INVERSE: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); + + Identifier in = getFirstExpr().getOutput(); + if(in.dimsKnown() && in.getDim1() != in.getDim2()) + raiseValidateError( + "Input to inv() must be square matrix -- given: a " + in.getDim1() + "x" + in.getDim2() + + " matrix.", conditional); + + output.setDimensions(in.getDim1(), in.getDim2()); + output.setBlocksize(in.getBlocksize()); + break; + + case SQRT_MATRIX_JAVA: - output.setValueType(id.getValueType()); - // output dimensions are unknown - output.setDimensions(-1, -1); - output.setBlocksize(-1); - output.setDataType(DataType.MATRIX); - break; - - case IQM: - /* - * Usage: iqm = InterQuartileMean(A,W); iqm = InterQuartileMean(A); - */ - if (getSecondExpr() != null){ - checkNumParameters(2); - } - else { checkNumParameters(1); - } - checkMatrixParam(getFirstExpr()); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); + Identifier sqrt = getFirstExpr().getOutput(); + if(sqrt.dimsKnown() && sqrt.getDim1() != sqrt.getDim2()) + raiseValidateError( + "Input to sqrtMatrix() must be square matrix -- given: a " + sqrt.getDim1() + "x" + + sqrt.getDim2() + " matrix.", conditional); + output.setDimensions(sqrt.getDim1(), sqrt.getDim2()); + output.setBlocksize(sqrt.getBlocksize()); + break; - if (getSecondExpr() != null) { - // i.e., second input is weight vector - checkMatrixParam(getSecondExpr()); - checkMatchingDimensions(getFirstExpr(), getSecondExpr()); - } + case CHOLESKY: { + // A = L%*%t(L) where L is the lower triangular matrix + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); - // Output is a scalar - output.setValueType(id.getValueType()); - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setDataType(DataType.SCALAR); - - break; - - case ISNA: - case ISNAN: - case ISINF: - checkNumParameters(1); - checkMatrixScalarParam(getFirstExpr()); - output.setDataType(id.getDataType()); - output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize (id.getBlocksize()); - //TODO set output type to boolean when supported - output.setValueType(id.getValueType()); - break; - - case MEDIAN: - checkNumParameters((getSecondExpr()!=null) ? 2 : 1); - checkMatrixParam(getFirstExpr()); - - if (getSecondExpr() != null) { - // i.e., second input is weight vector - checkMatrixParam(getSecondExpr()); - checkMatchingDimensions(getFirstExpr(), getSecondExpr()); - } + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); - // Output is a scalar - output.setValueType(id.getValueType()); - output.setDimensions(0, 0); - output.setBlocksize(0); - output.setDataType(DataType.SCALAR); - - break; - - case SAMPLE: - { - Expression[] in = getAllExpr(); - - for(Expression e : in) - checkScalarParam(e); - - if (in[0].getOutput().getValueType() != ValueType.FP64 && in[0].getOutput().getValueType() != ValueType.INT64) - throw new LanguageException("First argument to sample() must be a number."); - if (in[1].getOutput().getValueType() != ValueType.FP64 && in[1].getOutput().getValueType() != ValueType.INT64) - throw new LanguageException("Second argument to sample() must be a number."); - - boolean check = false; - if ( isConstant(in[0]) && isConstant(in[1]) ) - { - long range = ((ConstIdentifier)in[0]).getLongValue(); - long size = ((ConstIdentifier)in[1]).getLongValue(); - if ( range < size ) - check = true; - } - - if(in.length == 4 ) - { - checkNumParameters(4); - if (in[3].getOutput().getValueType() != ValueType.INT64) - throw new LanguageException("Fourth argument, seed, to sample() must be an integer value."); - if (in[2].getOutput().getValueType() != ValueType.BOOLEAN ) - throw new LanguageException("Third argument to sample() must either denote replacement policy (boolean) or seed (integer)."); - } - else if(in.length == 3) - { - checkNumParameters(3); - if (in[2].getOutput().getValueType() != ValueType.BOOLEAN - && in[2].getOutput().getValueType() != ValueType.INT64 ) - throw new LanguageException("Third argument to sample() must either denote replacement policy (boolean) or seed (integer)."); + Identifier inA = getFirstExpr().getOutput(); + if(inA.dimsKnown() && inA.getDim1() != inA.getDim2()) + raiseValidateError( + "Input to cholesky() must be square matrix -- given: a " + inA.getDim1() + "x" + inA.getDim2() + + " matrix.", conditional); + + output.setDimensions(inA.getDim1(), inA.getDim2()); + output.setBlocksize(inA.getBlocksize()); + break; } - - if ( check && in.length >= 3 - && isConstant(in[2]) - && in[2].getOutput().getValueType() == ValueType.BOOLEAN - && !((BooleanIdentifier)in[2]).getValue() ) - throw new LanguageException("Sample (size=" + ((ConstIdentifier)in[0]).getLongValue() - + ") larger than population (size=" + ((ConstIdentifier)in[1]).getLongValue() - + ") can only be generated with replacement."); - - // Output is a column vector - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - - if ( isConstant(in[1]) ) - output.setDimensions(((ConstIdentifier)in[1]).getLongValue(), 1); - else - output.setDimensions(-1, 1); - setBlocksize(id.getBlocksize()); - - break; - } - case SEQ: - - //basic parameter validation - checkScalarParam(getFirstExpr()); - checkScalarParam(getSecondExpr()); - if ( getThirdExpr() != null ) { + + case OUTER: + Identifier id2 = this.getSecondExpr().getOutput(); + + //check input types and characteristics checkNumParameters(3); + checkMatrixParam(getFirstExpr()); + checkMatrixParam(getSecondExpr()); checkScalarParam(getThirdExpr()); - } - else - checkNumParameters(2); - - // constant propagation (from, to, incr) - if( !conditional ) { - if( getFirstExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getFirstExpr()).getName()) ) - _args[0] = constVars.get(((DataIdentifier)getFirstExpr()).getName()); - if( getSecondExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getSecondExpr()).getName()) ) - _args[1] = constVars.get(((DataIdentifier)getSecondExpr()).getName()); - if( getThirdExpr()!=null && getThirdExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getThirdExpr()).getName()) ) - _args[2] = constVars.get(((DataIdentifier)getThirdExpr()).getName()); - } - - // check if dimensions can be inferred - long dim1=-1, dim2=1; - if ( isConstant(getFirstExpr()) && isConstant(getSecondExpr()) && (getThirdExpr() != null ? isConstant(getThirdExpr()) : true) ) { - double from, to, incr; - try { - from = getDoubleValue(getFirstExpr()); - to = getDoubleValue(getSecondExpr()); - - // Setup the value of increment - // default value: 1 if from <= to; -1 if from > to - if(getThirdExpr() == null) { - expandArguments(); - _args[2] = new DoubleIdentifier(((from > to) ? -1.0 : 1.0), this); - } - incr = getDoubleValue(getThirdExpr()); - - } - catch (LanguageException e) { - throw new LanguageException("Arguments for seq() must be numeric."); + checkValueTypeParam(getThirdExpr(), ValueType.STRING); + if(id.getDim2() > 1 || id2.getDim1() > 1) { + raiseValidateError( + "Outer vector operations require a common dimension of one: " + id.getDim1() + "x" + + id.getDim2() + " o " + id2.getDim1() + "x" + id2.getDim2() + ".", false); } - if( (from > to) && (incr >= 0) ) - throw new LanguageException("Wrong sign for the increment in a call to seq()"); - - // Both end points of the range must included i.e., [from,to] both inclusive. - // Note that, "to" is included only if (to-from) is perfectly divisible by incr - // For example, seq(0,1,0.5) produces (0.0 0.5 1.0) whereas seq(0,1,0.6) produces only (0.0 0.6) but not (0.0 0.6 1.0) - dim1 = UtilFunctions.getSeqLength(from, to, incr); - } - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - output.setDimensions(dim1, dim2); - output.setBlocksize(0); - break; - - case SOLVE: - checkNumParameters(2); - checkMatrixParam(getFirstExpr()); - checkMatrixParam(getSecondExpr()); - - if ( getSecondExpr().getOutput().dimsKnown() && !is1DMatrix(getSecondExpr()) ) - raiseValidateError("Second input to solve() must be a vector", conditional); - - if ( getFirstExpr().getOutput().dimsKnown() && getSecondExpr().getOutput().dimsKnown() && - getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1() && - getFirstExpr().getOutput().getDim1() != getFirstExpr().getOutput().getDim2()) - raiseValidateError("Dimension mismatch in a call to solve()", conditional); - - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - output.setDimensions(getFirstExpr().getOutput().getDim2(), 1); - output.setBlocksize(0); - break; - - case INVERSE: - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - - Identifier in = getFirstExpr().getOutput(); - if(in.dimsKnown() && in.getDim1() != in.getDim2()) - raiseValidateError("Input to inv() must be square matrix -- given: a " + in.getDim1() + "x" + in.getDim2() + " matrix.", conditional); - - output.setDimensions(in.getDim1(), in.getDim2()); - output.setBlocksize(in.getBlocksize()); - break; - - case SQRT_MATRIX_JAVA: - - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - Identifier sqrt = getFirstExpr().getOutput(); - if(sqrt.dimsKnown() && sqrt.getDim1() != sqrt.getDim2()) - raiseValidateError("Input to sqrtMatrix() must be square matrix -- given: a " + sqrt.getDim1() + "x" + sqrt.getDim2() + " matrix.", conditional); - output.setDimensions( sqrt.getDim1(), sqrt.getDim2()); - output.setBlocksize( sqrt.getBlocksize()); - break; - - case CHOLESKY: - { - // A = L%*%t(L) where L is the lower triangular matrix - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - - Identifier inA = getFirstExpr().getOutput(); - if(inA.dimsKnown() && inA.getDim1() != inA.getDim2()) - raiseValidateError("Input to cholesky() must be square matrix -- given: a " + inA.getDim1() + "x" + inA.getDim2() + " matrix.", conditional); - - output.setDimensions(inA.getDim1(), inA.getDim2()); - output.setBlocksize(inA.getBlocksize()); - break; - } + //set output characteristics + output.setDataType(id.getDataType()); + output.setDimensions(id.getDim1(), id2.getDim2()); + output.setBlocksize(id.getBlocksize()); + break; - case OUTER: - Identifier id2 = this.getSecondExpr().getOutput(); - - //check input types and characteristics - checkNumParameters(3); - checkMatrixParam(getFirstExpr()); - checkMatrixParam(getSecondExpr()); - checkScalarParam(getThirdExpr()); - checkValueTypeParam(getThirdExpr(), ValueType.STRING); - if( id.getDim2() > 1 || id2.getDim1()>1 ) { - raiseValidateError("Outer vector operations require a common dimension of one: " + - id.getDim1()+"x"+id.getDim2()+" o "+id2.getDim1()+"x"+id2.getDim2()+".", false); - } - - //set output characteristics - output.setDataType(id.getDataType()); - output.setDimensions(id.getDim1(), id2.getDim2()); - output.setBlocksize(id.getBlocksize()); - break; - - case BIASADD: - case BIASMULT: - { - Expression input = _args[0]; - Expression bias = _args[1]; - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - output.setDimensions(input.getOutput().getDim1(), input.getOutput().getDim2()); - output.setBlocksize(input.getOutput().getBlocksize()); - checkMatrixParam(input); - checkMatrixParam(bias); - break; - } - case CONV2D: - case CONV2D_BACKWARD_FILTER: - case CONV2D_BACKWARD_DATA: - case MAX_POOL: - case AVG_POOL: - case MAX_POOL_BACKWARD: - case AVG_POOL_BACKWARD: - { - // At DML level: - // output = conv2d(input, filter, input_shape=[1, 3, 2, 2], filter_shape=[1, 3, 2, 2], - // strides=[1, 1], padding=[1,1]) - // - // Converted to following in constructor (only supported NCHW): - // output = conv2d(input, filter, stride1, stride2, padding1,padding2, - // input_shape1, input_shape2, input_shape3, input_shape4, - // filter_shape1, filter_shape2, filter_shape3, filter_shape4) - // - // Similarly, - // conv2d_backward_filter and conv2d_backward_data - Expression input = _args[0]; // For conv2d_backward_filter, this is input and for conv2d_backward_data, this is filter - - Expression input2 = null; - if(!(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL)) { - input2 = _args[1]; // For conv2d_backward functions, this is dout - checkMatrixParam(input2); - } - output.setDataType(DataType.MATRIX); - output.setValueType(ValueType.FP64); - output.setBlocksize(input.getOutput().getBlocksize()); - - if(this.getOpCode() == Builtins.MAX_POOL_BACKWARD || this.getOpCode() == Builtins.AVG_POOL_BACKWARD) { + case BIASADD: + case BIASMULT: { + Expression input = _args[0]; + Expression bias = _args[1]; + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); output.setDimensions(input.getOutput().getDim1(), input.getOutput().getDim2()); + output.setBlocksize(input.getOutput().getBlocksize()); + checkMatrixParam(input); + checkMatrixParam(bias); + break; } - else { - // stride1, stride2, padding1, padding2, numImg, numChannels, imgSize, imgSize, - // filter_shape1=1, filter_shape2=1, filterSize/poolSize1, filterSize/poolSize1 - try { - int start = 2; - if(!(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL)) { - start = 1; - } - long stride_h = (long) getDoubleValue(_args[start++]); - long stride_w = (long) getDoubleValue(_args[start++]); - long pad_h = (long) getDoubleValue(_args[start++]); - long pad_w = (long) getDoubleValue(_args[start++]); - long N = (long) getDoubleValue(_args[start++]); - long C = (long) getDoubleValue(_args[start++]); - long H = (long) getDoubleValue(_args[start++]); - long W = (long) getDoubleValue(_args[start++]); - long K = -1; - if(!(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL)) { - K = (long) getDoubleValue(_args[start]); - } - start++; start++; // Increment index for K and C - long R = (long) getDoubleValue(_args[start++]); - long S = (long) getDoubleValue(_args[start++]); - - if(this.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER) { - output.setDimensions(K, C*R*S); - } - else if(this.getOpCode() == Builtins.CONV2D_BACKWARD_DATA) { - output.setDimensions(N, C*H*W); - } - else if(H > 0 && W > 0 && stride_h > 0 && stride_w > 0 && pad_h >= 0 && pad_w >= 0 && R > 0 && S > 0) { - long P = DnnUtils.getP(H, R, stride_h, pad_h); - long Q = DnnUtils.getQ(W, S, stride_w, pad_w); - - // Try to set both rows and columns - if(this.getOpCode() == Builtins.CONV2D) - output.setDimensions(N, K*P*Q); - else if(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL) - output.setDimensions(N, C*P*Q); - else - throw new LanguageException(""); + case CONV2D: + case CONV2D_BACKWARD_FILTER: + case CONV2D_BACKWARD_DATA: + case MAX_POOL: + case AVG_POOL: + case MAX_POOL_BACKWARD: + case AVG_POOL_BACKWARD: { + // At DML level: + // output = conv2d(input, filter, input_shape=[1, 3, 2, 2], filter_shape=[1, 3, 2, 2], + // strides=[1, 1], padding=[1,1]) + // + // Converted to following in constructor (only supported NCHW): + // output = conv2d(input, filter, stride1, stride2, padding1,padding2, + // input_shape1, input_shape2, input_shape3, input_shape4, + // filter_shape1, filter_shape2, filter_shape3, filter_shape4) + // + // Similarly, + // conv2d_backward_filter and conv2d_backward_data + Expression input = _args[0]; // For conv2d_backward_filter, this is input and for conv2d_backward_data, this is filter + + Expression input2 = null; + if(!(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL)) { + input2 = _args[1]; // For conv2d_backward functions, this is dout + checkMatrixParam(input2); + } + output.setDataType(DataType.MATRIX); + output.setValueType(ValueType.FP64); + output.setBlocksize(input.getOutput().getBlocksize()); + + if(this.getOpCode() == Builtins.MAX_POOL_BACKWARD || this.getOpCode() == Builtins.AVG_POOL_BACKWARD) { + output.setDimensions(input.getOutput().getDim1(), input.getOutput().getDim2()); + } + else { + // stride1, stride2, padding1, padding2, numImg, numChannels, imgSize, imgSize, + // filter_shape1=1, filter_shape2=1, filterSize/poolSize1, filterSize/poolSize1 + try { + int start = 2; + if(!(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL)) { + start = 1; + } + long stride_h = (long) getDoubleValue(_args[start++]); + long stride_w = (long) getDoubleValue(_args[start++]); + long pad_h = (long) getDoubleValue(_args[start++]); + long pad_w = (long) getDoubleValue(_args[start++]); + long N = (long) getDoubleValue(_args[start++]); + long C = (long) getDoubleValue(_args[start++]); + long H = (long) getDoubleValue(_args[start++]); + long W = (long) getDoubleValue(_args[start++]); + long K = -1; + if(!(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL)) { + K = (long) getDoubleValue(_args[start]); + } + start++; + start++; // Increment index for K and C + long R = (long) getDoubleValue(_args[start++]); + long S = (long) getDoubleValue(_args[start++]); + + if(this.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER) { + output.setDimensions(K, C * R * S); + } + else if(this.getOpCode() == Builtins.CONV2D_BACKWARD_DATA) { + output.setDimensions(N, C * H * W); + } + else if(H > 0 && W > 0 && stride_h > 0 && stride_w > 0 && pad_h >= 0 && pad_w >= 0 && R > 0 && + S > 0) { + long P = DnnUtils.getP(H, R, stride_h, pad_h); + long Q = DnnUtils.getQ(W, S, stride_w, pad_w); + + // Try to set both rows and columns + if(this.getOpCode() == Builtins.CONV2D) + output.setDimensions(N, K * P * Q); + else if(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL) + output.setDimensions(N, C * P * Q); + else + throw new LanguageException(""); + } + else { + // Since columns cannot be computed, set only rows + if(this.getOpCode() == Builtins.CONV2D) + output.setDimensions(input.getOutput().getDim1(), -1); + else if(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL) + output.setDimensions(input.getOutput().getDim1(), -1); + else + throw new LanguageException(""); + } } - else { - // Since columns cannot be computed, set only rows - if(this.getOpCode() == Builtins.CONV2D) - output.setDimensions(input.getOutput().getDim1(), -1); - else if(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AVG_POOL) - output.setDimensions(input.getOutput().getDim1(), -1); - else - throw new LanguageException(""); + catch(Exception e) { + output.setDimensions(-1, + -1); // To make sure that output dimensions are not incorrect even if getDoubleValue doesnot return value } } - catch(Exception e) { - output.setDimensions(-1, -1); // To make sure that output dimensions are not incorrect even if getDoubleValue doesnot return value - } + checkMatrixParam(input); + if(input2 != null) + checkMatrixParam(input2); + break; } - checkMatrixParam(input); - if(input2 != null) - checkMatrixParam(input2); - break; - } - case TIME: - checkNumParameters(0); - // Output of TIME() is scalar and long - output.setDataType(DataType.SCALAR); - output.setValueType(ValueType.INT64); - output.setDimensions(0, 0); - output.setBlocksize(0); - break; - - case DROP_INVALID_TYPE: - case VALUE_SWAP: - case FRAME_ROW_REPLICATE: - checkNumParameters(2); - checkMatrixFrameParam(getFirstExpr()); - checkMatrixFrameParam(getSecondExpr()); - output.setDataType(DataType.FRAME); - output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(ValueType.STRING); - break; - - case DROP_INVALID_LENGTH: - checkNumParameters(2); - checkMatrixFrameParam(getFirstExpr()); - checkMatrixFrameParam(getSecondExpr()); - output.setDataType(DataType.FRAME); - output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(id.getValueType()); - break; - case APPLY_SCHEMA: + case TIME: + checkNumParameters(0); + // Output of TIME() is scalar and long + output.setDataType(DataType.SCALAR); + output.setValueType(ValueType.INT64); + output.setDimensions(0, 0); + output.setBlocksize(0); + break; + + case DROP_INVALID_TYPE: + case VALUE_SWAP: + case FRAME_ROW_REPLICATE: checkNumParameters(2); checkMatrixFrameParam(getFirstExpr()); checkMatrixFrameParam(getSecondExpr()); output.setDataType(DataType.FRAME); output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize (id.getBlocksize()); - break; - case MAP: - checkNumParameters(getThirdExpr() != null ? 3 : 2); - checkMatrixFrameParam(getFirstExpr()); - checkScalarParam(getSecondExpr()); - if(getThirdExpr() != null) - checkScalarParam(getThirdExpr()); // margin - output.setDataType(DataType.FRAME); - if(_args[1].getText().contains("jaccardSim")) { - output.setDimensions(id.getDim1(), id.getDim1()); - output.setValueType(ValueType.FP64); - } - else { - output.setDimensions(id.getDim1(), id.getDim2()); + output.setBlocksize(id.getBlocksize()); output.setValueType(ValueType.STRING); - } - break; - case LOCAL: - if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_LOCAL_COMMAND){ - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); + break; + + case DROP_INVALID_LENGTH: + checkNumParameters(2); + checkMatrixFrameParam(getFirstExpr()); + checkMatrixFrameParam(getSecondExpr()); + output.setDataType(DataType.FRAME); output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize (id.getBlocksize()); + output.setBlocksize(id.getBlocksize()); output.setValueType(id.getValueType()); - } - else - raiseValidateError("Local instruction not allowed in dml script"); - case COMPRESS: - case DECOMPRESS: - if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_COMPRESS_COMMAND){ - checkNumParameters(1); + break; + case APPLY_SCHEMA: + checkNumParameters(2); checkMatrixFrameParam(getFirstExpr()); - output.setDataType(getFirstExpr().getOutput().getDataType()); + checkMatrixFrameParam(getSecondExpr()); + output.setDataType(DataType.FRAME); output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(id.getValueType()); - } - else - raiseValidateError("The compress or decompress instruction is not allowed in dml scripts"); - break; - case GET_CATEGORICAL_MASK: - checkNumParameters(2); - checkFrameParam(getFirstExpr()); - checkScalarParam(getSecondExpr()); - output.setDataType(DataType.MATRIX); - output.setDimensions(1, -1); - output.setBlocksize( id.getBlocksize()); - output.setValueType(ValueType.FP64); - break; - case QUANTIZE_COMPRESS: - if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_QUANTIZE_COMPRESS_COMMAND) { + output.setBlocksize(id.getBlocksize()); + break; + case MAP: + checkNumParameters(getThirdExpr() != null ? 3 : 2); + checkMatrixFrameParam(getFirstExpr()); + checkScalarParam(getSecondExpr()); + if(getThirdExpr() != null) + checkScalarParam(getThirdExpr()); // margin + output.setDataType(DataType.FRAME); + if(_args[1].getText().contains("jaccardSim")) { + output.setDimensions(id.getDim1(), id.getDim1()); + output.setValueType(ValueType.FP64); + } + else { + output.setDimensions(id.getDim1(), id.getDim2()); + output.setValueType(ValueType.STRING); + } + break; + case LOCAL: + if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_LOCAL_COMMAND) { + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(id.getDim1(), id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + } + else + raiseValidateError("Local instruction not allowed in dml script"); + case COMPRESS: + case DECOMPRESS: + if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_COMPRESS_COMMAND) { + checkNumParameters(1); + checkMatrixFrameParam(getFirstExpr()); + output.setDataType(getFirstExpr().getOutput().getDataType()); + output.setDimensions(id.getDim1(), id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(id.getValueType()); + } + else + raiseValidateError("The compress or decompress instruction is not allowed in dml scripts"); + break; + case GET_CATEGORICAL_MASK: checkNumParameters(2); - Expression firstExpr = getFirstExpr(); - Expression secondExpr = getSecondExpr(); - - checkMatrixParam(getFirstExpr()); - - if(secondExpr != null) { - // check if scale factor is a scalar, vector or matrix - checkMatrixScalarParam(secondExpr); - // if scale factor is a vector or matrix, make sure it has an appropriate shape - if(secondExpr.getOutput().getDataType() != DataType.SCALAR) { - if(is1DMatrix(secondExpr)) { - long vectorLength = secondExpr.getOutput().getDim1(); - if(vectorLength != firstExpr.getOutput().getDim1()) { - raiseValidateError( - "The length of the row-wise scale factor vector must match the number of rows in the matrix."); + checkFrameParam(getFirstExpr()); + checkScalarParam(getSecondExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(1, -1); + output.setBlocksize(id.getBlocksize()); + output.setValueType(ValueType.FP64); + break; + case QUANTIZE_COMPRESS: + if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_QUANTIZE_COMPRESS_COMMAND) { + checkNumParameters(2); + Expression firstExpr = getFirstExpr(); + Expression secondExpr = getSecondExpr(); + + checkMatrixParam(getFirstExpr()); + + if(secondExpr != null) { + // check if scale factor is a scalar, vector or matrix + checkMatrixScalarParam(secondExpr); + // if scale factor is a vector or matrix, make sure it has an appropriate shape + if(secondExpr.getOutput().getDataType() != DataType.SCALAR) { + if(is1DMatrix(secondExpr)) { + long vectorLength = secondExpr.getOutput().getDim1(); + if(vectorLength != firstExpr.getOutput().getDim1()) { + raiseValidateError( + "The length of the row-wise scale factor vector must match the number of rows in the matrix."); + } + } + else { + checkMatchingDimensions(firstExpr, secondExpr); } - } - else { - checkMatchingDimensions(firstExpr, secondExpr); } } } - } - else - raiseValidateError("The quantize_compress instruction not allowed in dml scripts"); - break; - - case ROW_COUNT_DISTINCT: - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); - output.setDimensions(id.getDim1(), 1); - output.setBlocksize (id.getBlocksize()); - output.setValueType(ValueType.INT64); - output.setNnz(id.getDim1()); - break; - - case COL_COUNT_DISTINCT: - checkNumParameters(1); - checkMatrixParam(getFirstExpr()); - output.setDataType(DataType.MATRIX); - output.setDimensions(1, id.getDim2()); - output.setBlocksize (id.getBlocksize()); - output.setValueType(ValueType.INT64); - output.setNnz(id.getDim2()); - break; - case EINSUM: - validateEinsum(output); - break; - default: - if( isMathFunction() ) { - checkMathFunctionParam(); - //unary operations - if( getSecondExpr() == null ) { - output.setDataType(id.getDataType()); - output.setValueType((output.getDataType()==DataType.SCALAR - && getOpCode()==Builtins.ABS)?id.getValueType():ValueType.FP64 ); - output.setDimensions(id.getDim1(), id.getDim2()); - output.setBlocksize(id.getBlocksize()); + else + raiseValidateError("The quantize_compress instruction not allowed in dml scripts"); + break; + + case ROW_COUNT_DISTINCT: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(id.getDim1(), 1); + output.setBlocksize(id.getBlocksize()); + output.setValueType(ValueType.INT64); + output.setNnz(id.getDim1()); + break; + + case COL_COUNT_DISTINCT: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(1, id.getDim2()); + output.setBlocksize(id.getBlocksize()); + output.setValueType(ValueType.INT64); + output.setNnz(id.getDim2()); + break; + case EINSUM: + validateEinsum(output); + break; + default: + if(isMathFunction()) { + checkMathFunctionParam(); + //unary operations + if(getSecondExpr() == null) { + output.setDataType(id.getDataType()); + output.setValueType((output.getDataType() == DataType.SCALAR && + getOpCode() == Builtins.ABS) ? id.getValueType() : ValueType.FP64); + output.setDimensions(id.getDim1(), id.getDim2()); + output.setBlocksize(id.getBlocksize()); + } + //binary operations + else { + setBinaryOutputProperties(output); + // override computed value type for special cases + if(getOpCode() == Builtins.LOG) + output.setValueType(ValueType.FP64); + } } - //binary operations else { - setBinaryOutputProperties(output); - // override computed value type for special cases - if( getOpCode() == Builtins.LOG ) - output.setValueType(ValueType.FP64); + // always unconditional (because unsupported operation) + Builtins op = getOpCode(); + if(op == Builtins.EIGEN || op == Builtins.LU || op == Builtins.QR || op == Builtins.SVD || + op == Builtins.LSTM || op == Builtins.LSTM_BACKWARD || op == Builtins.BATCH_NORM2D || + op == Builtins.BATCH_NORM2D_BACKWARD) + raiseValidateError("Function " + op + " needs to be called with multi-return assignment.", + false, LanguageErrorCodes.INVALID_PARAMETERS); + else + raiseValidateError("Unsupported function " + op, false, LanguageErrorCodes.INVALID_PARAMETERS); } - } - else { - // always unconditional (because unsupported operation) - Builtins op = getOpCode(); - if( op==Builtins.EIGEN || op==Builtins.LU || op==Builtins.QR || op==Builtins.SVD - || op==Builtins.LSTM || op==Builtins.LSTM_BACKWARD - || op==Builtins.BATCH_NORM2D || op==Builtins.BATCH_NORM2D_BACKWARD) - raiseValidateError("Function "+op+" needs to be called with multi-return assignment.", false, LanguageErrorCodes.INVALID_PARAMETERS); - else - raiseValidateError("Unsupported function "+op, false, LanguageErrorCodes.INVALID_PARAMETERS); - } } } - private void validateEinsum(DataIdentifier output){ + private void validateEinsum(DataIdentifier output) { if(getSecondExpr() == null) raiseValidateError("Einsum: at least one input matrix required", false, - LanguageErrorCodes.INVALID_PARAMETERS); + LanguageErrorCodes.INVALID_PARAMETERS); if(!(getFirstExpr() instanceof StringIdentifier)) raiseValidateError("Einsum: first argument has to be equation str", false, - LanguageErrorCodes.INVALID_PARAMETERS); + LanguageErrorCodes.INVALID_PARAMETERS); - String equationString = ((StringIdentifier)getFirstExpr()).getValue(); + String equationString = ((StringIdentifier) getFirstExpr()).getValue(); - if (equationString.length() == 0) raiseValidateError("Einsum: equation str too short", false, LanguageErrorCodes.INVALID_PARAMETERS); - if (equationString.charAt(0) == '-' || equationString.charAt(0) == ',') raiseValidateError("Einsum: equation str invalid", false, LanguageErrorCodes.INVALID_PARAMETERS); + if(equationString.length() == 0) + raiseValidateError("Einsum: equation str too short", false, LanguageErrorCodes.INVALID_PARAMETERS); + if(equationString.charAt(0) == '-' || equationString.charAt(0) == ',') + raiseValidateError("Einsum: equation str invalid", false, LanguageErrorCodes.INVALID_PARAMETERS); Expression[] expressions = getAllExpr(); boolean allDimsKnown = true; LinkedList matrixBlocks = new LinkedList<>(); - for (int i=1;i e.getOutput().getDataType().isScalar()) ? DataType.SCALAR : DataType.MATRIX; - Expression firstM = dt.isMatrix() ? Arrays.stream(getAllExpr()).filter( - e -> e.getOutput().getDataType().isMatrix()).findFirst().get() : null; + DataType dt = Arrays.stream(getAllExpr()) + .allMatch(e -> e.getOutput().getDataType().isScalar()) ? DataType.SCALAR : DataType.MATRIX; + Expression firstM = dt.isMatrix() ? Arrays.stream(getAllExpr()) + .filter(e -> e.getOutput().getDataType().isMatrix()).findFirst().get() : null; ValueType vt = dt.isMatrix() ? ValueType.FP64 : ValueType.INT64; - for( Expression e : getAllExpr() ) { + for(Expression e : getAllExpr()) { vt = computeValueType(e, e.getOutput().getValueType(), vt, true); - if( e.getOutput().getDataType().isMatrix() ) + if(e.getOutput().getDataType().isMatrix()) checkMatchingDimensions(firstM, e, true); } output.setDataType(dt); output.setValueType(vt); output.setDimensions(dt.isMatrix() ? firstM.getOutput().getDim1() : 0, dt.isMatrix() ? firstM.getOutput().getDim2() : 0); - output.setBlocksize (dt.isMatrix() ? firstM.getOutput().getBlocksize() : 0); + output.setBlocksize(dt.isMatrix() ? firstM.getOutput().getBlocksize() : 0); } - + private void expandArguments() { - - if ( _args == null ) { + + if(_args == null) { _args = new Expression[1]; return; } - Expression [] temp = _args.clone(); + Expression[] temp = _args.clone(); _args = new Expression[_args.length + 1]; System.arraycopy(temp, 0, _args, 0, temp.length); } - + @Override public boolean multipleReturns() { return _opcode.isMultiReturn(); } private static boolean isConstant(Expression expr) { - return ( expr != null && expr instanceof ConstIdentifier ); + return (expr != null && expr instanceof ConstIdentifier); } - + private static double getDoubleValue(Expression expr) { - if ( expr instanceof DoubleIdentifier ) - return ((DoubleIdentifier)expr).getValue(); - else if ( expr instanceof IntIdentifier) - return ((IntIdentifier)expr).getValue(); + if(expr instanceof DoubleIdentifier) + return ((DoubleIdentifier) expr).getValue(); + else if(expr instanceof IntIdentifier) + return ((IntIdentifier) expr).getValue(); else throw new LanguageException("Expecting a numeric value."); } - + private boolean isMathFunction() { - switch (this.getOpCode()) { - case COS: - case SIN: - case TAN: - case ACOS: - case ASIN: - case ATAN: - case COSH: - case SINH: - case TANH: - case SIGN: - case SQRT: - case ABS: - case LOG: - case EXP: - case ROUND: - case CEIL: - case FLOOR: - case MEDIAN: - case XOR: - case BITWAND: - case BITWOR: - case BITWXOR: - case BITWSHIFTL: - case BITWSHIFTR: - return true; - default: - return false; + switch(this.getOpCode()) { + case COS: + case SIN: + case TAN: + case ACOS: + case ASIN: + case ATAN: + case COSH: + case SINH: + case TANH: + case SIGN: + case SQRT: + case ABS: + case LOG: + case EXP: + case ROUND: + case CEIL: + case FLOOR: + case MEDIAN: + case XOR: + case BITWAND: + case BITWOR: + case BITWXOR: + case BITWSHIFTL: + case BITWSHIFTR: + return true; + default: + return false; } } private void checkMathFunctionParam() { - switch (this.getOpCode()) { - case COS: - case SIN: - case TAN: - case ACOS: - case ASIN: - case ATAN: - case COSH: - case SINH: - case TANH: - case SIGN: - case SQRT: - case ABS: - case EXP: - case ROUND: - case CEIL: - case FLOOR: - case MEDIAN: - checkNumParameters(1); - break; - case LOG: - if (getSecondExpr() != null) { - checkNumParameters(2); - } - else { - checkNumParameters(1); - } - break; - default: - //always unconditional - raiseValidateError("Unknown math function "+ this.getOpCode(), false); + switch(this.getOpCode()) { + case COS: + case SIN: + case TAN: + case ACOS: + case ASIN: + case ATAN: + case COSH: + case SINH: + case TANH: + case SIGN: + case SQRT: + case ABS: + case EXP: + case ROUND: + case CEIL: + case FLOOR: + case MEDIAN: + checkNumParameters(1); + break; + case LOG: + if(getSecondExpr() != null) { + checkNumParameters(2); + } + else { + checkNumParameters(1); + } + break; + default: + //always unconditional + raiseValidateError("Unknown math function " + this.getOpCode(), false); } } @Override public String toString() { StringBuilder sb = new StringBuilder(_opcode.toString() + "("); - if (_args != null) { - for (int i = 0; i < _args.length; i++) { - if (i > 0) { + if(_args != null) { + for(int i = 0; i < _args.length; i++) { + if(i > 0) { sb.append(","); } sb.append(_args[i].toString()); @@ -2323,11 +2373,11 @@ public String toString() { // third part of expression IS NOT a variable -- it is the OP to be applied public VariableSet variablesRead() { VariableSet result = new VariableSet(); - - for(int i=0; i<_args.length; i++) { + + for(int i = 0; i < _args.length; i++) { result.addVariables(_args[i].variablesRead()); } - + return result; } @@ -2339,57 +2389,62 @@ public VariableSet variablesUpdated() { } protected void checkNumParameters(int count) { //always unconditional - if (getFirstExpr() == null && _args.length > 0) { + if(getFirstExpr() == null && _args.length > 0) { raiseValidateError("Missing argument for function " + this.getOpCode(), false, - LanguageErrorCodes.INVALID_PARAMETERS); + LanguageErrorCodes.INVALID_PARAMETERS); } // Not sure the rationale for the first two if loops, but will keep them for backward compatibility - if (((count == 1) && (getSecondExpr() != null || getThirdExpr() != null)) - || ((count == 2) && (getThirdExpr() != null))) { - raiseValidateError("Invalid number of arguments for function " + this.getOpCode().toString().toLowerCase() - + "(). This function only takes 1 or 2 arguments.", false); - } else if (((count == 2) && (getSecondExpr() == null)) - || ((count == 3) && (getSecondExpr() == null || getThirdExpr() == null))) { + if(((count == 1) && (getSecondExpr() != null || getThirdExpr() != null)) || + ((count == 2) && (getThirdExpr() != null))) { + raiseValidateError("Invalid number of arguments for function " + this.getOpCode().toString().toLowerCase() + + "(). This function only takes 1 or 2 arguments.", false); + } + else if(((count == 2) && (getSecondExpr() == null)) || + ((count == 3) && (getSecondExpr() == null || getThirdExpr() == null))) { raiseValidateError("Missing argument for function " + this.getOpCode(), false, - LanguageErrorCodes.INVALID_PARAMETERS); - } else if(count > 0 && (_args == null || _args.length < count)) { + LanguageErrorCodes.INVALID_PARAMETERS); + } + else if(count > 0 && (_args == null || _args.length < count)) { raiseValidateError("Missing argument for function " + this.getOpCode(), false, - LanguageErrorCodes.INVALID_PARAMETERS); - } else if (count == 0 && (_args.length > 0 - || getSecondExpr() != null || getThirdExpr() != null)) { - raiseValidateError("Missing argument for function " + this.getOpCode() - + "(). This function doesn't take any arguments.", false); + LanguageErrorCodes.INVALID_PARAMETERS); + } + else if(count == 0 && (_args.length > 0 || getSecondExpr() != null || getThirdExpr() != null)) { + raiseValidateError( + "Missing argument for function " + this.getOpCode() + "(). This function doesn't take any arguments.", + false); } } protected void checkMatrixParam(Expression e) { - if (e.getOutput().getDataType() != DataType.MATRIX) { + if(e.getOutput().getDataType() != DataType.MATRIX) { raiseValidateError("Expected " + e.getText() + " to be a matrix argument for function " + - this.getOpCode().toString().toLowerCase() + "().", false); + this.getOpCode().toString().toLowerCase() + "().", false); } } - + protected void checkMatrixTensorParam(Expression e) { - if (e.getOutput().getDataType() != DataType.MATRIX) { + if(e.getOutput().getDataType() != DataType.MATRIX) { // Param is not a matrix // TODO get supported Operations form builtins - if (e.getOutput().getDataType() != DataType.TENSOR || getOpCode() != Builtins.SUM) { + if(e.getOutput().getDataType() != DataType.TENSOR || getOpCode() != Builtins.SUM) { // Param is also not a tensor, or the operation is not supported on tensor - raiseValidateError("Expected " + e.getText() + " to be a matrix or tensor argument for function " - + this.getOpCode().toString().toLowerCase() + "().", false); + raiseValidateError("Expected " + e.getText() + " to be a matrix or tensor argument for function " + + this.getOpCode().toString().toLowerCase() + "().", false); } } } protected void checkDataTypeParam(Expression e, DataType... dt) { //always unconditional - if( !ArrayUtils.contains(dt, e.getOutput().getDataType()) ) - raiseValidateError("Non-matching expected data type for function "+ getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(!ArrayUtils.contains(dt, e.getOutput().getDataType())) + raiseValidateError("Non-matching expected data type for function " + getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } protected void checkMatrixFrameParam(Expression e) { //always unconditional - if (e.getOutput().getDataType() != DataType.MATRIX && e.getOutput().getDataType() != DataType.FRAME) { - raiseValidateError("Expecting matrix or frame parameter for function "+ getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(e.getOutput().getDataType() != DataType.MATRIX && e.getOutput().getDataType() != DataType.FRAME) { + raiseValidateError("Expecting matrix or frame parameter for function " + getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } @@ -2399,127 +2454,129 @@ protected void checkFrameParam(Expression e) { LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } - + protected void checkMatrixScalarParam(Expression e) { //always unconditional - if (e.getOutput().getDataType() != DataType.MATRIX && e.getOutput().getDataType() != DataType.SCALAR) { - raiseValidateError("Expecting matrix or scalar parameter for function "+ getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(e.getOutput().getDataType() != DataType.MATRIX && e.getOutput().getDataType() != DataType.SCALAR) { + raiseValidateError("Expecting matrix or scalar parameter for function " + getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } - + private void checkScalarParam(Expression e) { //always unconditional - if (e.getOutput().getDataType() != DataType.SCALAR) { - raiseValidateError("Expecting scalar parameter for function " + getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(e.getOutput().getDataType() != DataType.SCALAR) { + raiseValidateError("Expecting scalar parameter for function " + getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } - + private void checkListParam(Expression e) { //always unconditional - if (e.getOutput().getDataType() != DataType.LIST) { - raiseValidateError("Expecting scalar parameter for function " + getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(e.getOutput().getDataType() != DataType.LIST) { + raiseValidateError("Expecting scalar parameter for function " + getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } - + @SuppressWarnings("unused") private void checkScalarFrameParam(Expression e) { //always unconditional - if (e.getOutput().getDataType() != DataType.SCALAR && e.getOutput().getDataType() != DataType.FRAME) { - raiseValidateError("Expecting scalar parameter for function " + getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(e.getOutput().getDataType() != DataType.SCALAR && e.getOutput().getDataType() != DataType.FRAME) { + raiseValidateError("Expecting scalar parameter for function " + getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } private void checkValueTypeParam(Expression e, ValueType vt) { //always unconditional - if (e.getOutput().getValueType() != vt) { - raiseValidateError("Expecting parameter of different value type " + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(e.getOutput().getValueType() != vt) { + raiseValidateError("Expecting parameter of different value type " + this.getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } - + protected void checkStringOrDataIdentifier(Expression e) { //always unconditional - if( !(e.getOutput().getDataType().isScalar() && e.getOutput().getValueType()==ValueType.STRING) - && !(e instanceof DataIdentifier && !(e instanceof IndexedIdentifier)) ) { - raiseValidateError("Expecting variable name or data identifier "+ getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(!(e.getOutput().getDataType().isScalar() && e.getOutput().getValueType() == ValueType.STRING) && + !(e instanceof DataIdentifier && !(e instanceof IndexedIdentifier))) { + raiseValidateError("Expecting variable name or data identifier " + getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } - + private static boolean is1DMatrix(Expression e) { - return (e.getOutput().getDim1() == 1 || e.getOutput().getDim2() == 1 ); + return (e.getOutput().getDim1() == 1 || e.getOutput().getDim2() == 1); } - + private static boolean dimsKnown(Expression e) { return (e.getOutput().getDim1() != -1 && e.getOutput().getDim2() != -1); } - + private void check1DMatrixParam(Expression e) { //always unconditional checkMatrixParam(e); - + // throw an exception, when e's output is NOT a one-dimensional matrix // the check must be performed only when the dimensions are known at compilation time - if ( dimsKnown(e) && !is1DMatrix(e)) { - raiseValidateError("Expecting one-dimensional matrix parameter for function " - + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if(dimsKnown(e) && !is1DMatrix(e)) { + raiseValidateError("Expecting one-dimensional matrix parameter for function " + this.getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } private void checkMatchingDimensions(Expression expr1, Expression expr2) { checkMatchingDimensions(expr1, expr2, false); } - + private void checkMatchingDimensions(Expression expr1, Expression expr2, boolean allowsMV) { checkMatchingDimensions(expr1, expr2, allowsMV, false); } - - private void checkMatchingDimensions(Expression expr1, Expression expr2, boolean allowsMV, boolean conditional) - { - if (expr1 != null && expr2 != null) { - + + private void checkMatchingDimensions(Expression expr1, Expression expr2, boolean allowsMV, boolean conditional) { + if(expr1 != null && expr2 != null) { + // if any matrix has unknown dimensions, simply return - if( expr1.getOutput().getDim1() == -1 || expr2.getOutput().getDim1() == -1 - ||expr1.getOutput().getDim2() == -1 || expr2.getOutput().getDim2() == -1 ) - { + if(expr1.getOutput().getDim1() == -1 || expr2.getOutput().getDim1() == -1 || + expr1.getOutput().getDim2() == -1 || expr2.getOutput().getDim2() == -1) { return; } - else if( (!allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1()) - || (allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1() && expr2.getOutput().getDim1() != 1) - || (!allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2()) - || (allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2() && expr2.getOutput().getDim2() != 1) ) - { - raiseValidateError("Mismatch in matrix dimensions of parameters for function " - + this.getOpCode(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); + else if((!allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1()) || + (allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1() && + expr2.getOutput().getDim1() != 1) || + (!allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2()) || + (allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2() && + expr2.getOutput().getDim2() != 1)) { + raiseValidateError("Mismatch in matrix dimensions of parameters for function " + this.getOpCode(), + conditional, LanguageErrorCodes.INVALID_PARAMETERS); } } } - - private void checkMatchingDimensionsQuantile() - { - if (getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1()) { - raiseValidateError("Mismatch in matrix dimensions for " - + this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS); + + private void checkMatchingDimensionsQuantile() { + if(getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1()) { + raiseValidateError("Mismatch in matrix dimensions for " + this.getOpCode(), false, + LanguageErrorCodes.INVALID_PARAMETERS); } } - public static BuiltinFunctionExpression getBuiltinFunctionExpression(ParserRuleContext ctx, - String functionName, ArrayList paramExprsPassed, String filename) { - - if (functionName == null || paramExprsPassed == null) + public static BuiltinFunctionExpression getBuiltinFunctionExpression(ParserRuleContext ctx, String functionName, + ArrayList paramExprsPassed, String filename) { + + if(functionName == null || paramExprsPassed == null) return null; - + // check if the function name is built-in function // (assign built-in function op if function is built-in - - - return (Builtins.contains(functionName, false, false) - && (paramExprsPassed.stream().anyMatch(p -> p.getName()==null) //at least one unnamed - || paramExprsPassed.size() == 0)) ? - new BuiltinFunctionExpression(ctx, Builtins.get(functionName), paramExprsPassed, filename) : null; + + return (Builtins.contains(functionName, false, false) && + (paramExprsPassed.stream().anyMatch(p -> p.getName() == null) //at least one unnamed + || paramExprsPassed.size() == 0)) ? new BuiltinFunctionExpression(ctx, Builtins.get(functionName), + paramExprsPassed, filename) : null; } - + /** * Convert a value type (double, int, or boolean) to a built-in function operator. - * + * * @param vt Value type ({@code ValueType.DOUBLE}, {@code ValueType.INT}, or {@code ValueType.BOOLEAN}). - * @return Built-in function operator ({@code Builtins.AS_DOUBLE}, - * {@code Builtins.AS_INT}, or {@code Builtins.AS_BOOLEAN}). + * @return Built-in function operator ({@code Builtins.AS_DOUBLE}, {@code Builtins.AS_INT}, or + * {@code Builtins.AS_BOOLEAN}). */ - public static Builtins getValueTypeCastOperator( ValueType vt ) { - switch( vt ) - { + public static Builtins getValueTypeCastOperator(ValueType vt) { + switch(vt) { case FP64: return Builtins.CAST_AS_DOUBLE; case INT64: @@ -2527,7 +2584,7 @@ public static Builtins getValueTypeCastOperator( ValueType vt ) { case BOOLEAN: return Builtins.CAST_AS_BOOLEAN; default: - throw new LanguageException("No cast for value type "+vt); + throw new LanguageException("No cast for value type " + vt); } } } diff --git a/src/main/java/org/apache/sysds/parser/DMLTranslator.java b/src/main/java/org/apache/sysds/parser/DMLTranslator.java index e14cfd31388..ef08657489b 100644 --- a/src/main/java/org/apache/sysds/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysds/parser/DMLTranslator.java @@ -93,8 +93,7 @@ import org.apache.sysds.runtime.instructions.Instruction; import org.apache.sysds.runtime.instructions.cp.VariableCPInstruction; -public class DMLTranslator -{ +public class DMLTranslator { private static final Log LOG = LogFactory.getLog(DMLTranslator.class.getName()); private DMLProgram _dmlProg; @@ -103,25 +102,24 @@ public DMLTranslator(DMLProgram dmlp) { //setup default size for unknown dimensions OptimizerUtils.resetDefaultSize(); //reinit rewriter according to opt level flags - Recompiler.reinitRecompiler(); + Recompiler.reinitRecompiler(); } public void validateParseTree(DMLProgram dmlp) { validateParseTree(dmlp, true); } - public void validateParseTree(DMLProgram dmlp, boolean inclFuns) - { + public void validateParseTree(DMLProgram dmlp, boolean inclFuns) { //STEP1: Pre-processing steps for validate - e.g., prepare read-after-write meta data boolean fWriteRead = prepareReadAfterWrite(dmlp, new HashMap()); //STEP2: Actual Validate - if( inclFuns ) { + if(inclFuns) { // handle functions in namespaces (current program has default namespace) - for (String namespaceKey : dmlp.getNamespaces().keySet()) { + for(String namespaceKey : dmlp.getNamespaces().keySet()) { // for each function defined in the namespace - for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) { - FunctionStatementBlock fblock = dmlp.getFunctionStatementBlock(namespaceKey,fname); + for(String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) { + FunctionStatementBlock fblock = dmlp.getFunctionStatementBlock(namespaceKey, fname); validateFunction(dmlp, fblock); } } @@ -130,22 +128,21 @@ public void validateParseTree(DMLProgram dmlp, boolean inclFuns) // handle regular blocks -- "main" program VariableSet vs = new VariableSet(); HashMap constVars = new HashMap<>(); - for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) { + for(int i = 0; i < dmlp.getNumStatementBlocks(); i++) { StatementBlock sb = dmlp.getStatementBlock(i); vs = sb.validate(dmlp, vs, constVars, fWriteRead); constVars = sb.getConstOut(); } //STEP3: Post-processing steps after validate - e.g., prepare read-after-write meta data - if( fWriteRead ) - { + if(fWriteRead) { //propagate size and datatypes into read prepareReadAfterWrite(dmlp, new HashMap<>()); //re-validate main program for datatype propagation vs = new VariableSet(); constVars = new HashMap<>(); - for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) { + for(int i = 0; i < dmlp.getNumStatementBlocks(); i++) { StatementBlock sb = dmlp.getStatementBlock(i); vs = sb.validate(dmlp, vs, constVars, fWriteRead); constVars = sb.getConstOut(); @@ -162,9 +159,9 @@ public void validateFunction(DMLProgram dmlp, FunctionStatementBlock fsb, boolea VariableSet vs = new VariableSet(); // add the input variables for the function to input variable list - FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0); - for (DataIdentifier currVar : fstmt.getInputParams()) { - if (currVar.getDataType() == DataType.SCALAR) + FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0); + for(DataIdentifier currVar : fstmt.getInputParams()) { + if(currVar.getDataType() == DataType.SCALAR) currVar.setDimensions(0, 0); vs.addVariable(currVar.getName(), currVar); } @@ -178,9 +175,9 @@ public void liveVariableAnalysis(DMLProgram dmlp) { public void liveVariableAnalysis(DMLProgram dmlp, boolean inclFuns) { // for each namespace, handle function statement blocks - if( inclFuns ) { - for (String namespaceKey : dmlp.getNamespaces().keySet()) { - for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) { + if(inclFuns) { + for(String namespaceKey : dmlp.getNamespaces().keySet()) { + for(String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) { FunctionStatementBlock fsb = dmlp.getFunctionStatementBlock(namespaceKey, fname); liveVariableAnalysisFunction(dmlp, fsb); } @@ -194,15 +191,15 @@ public void liveVariableAnalysis(DMLProgram dmlp, boolean inclFuns) { // handle function inlining dmlp.setStatementBlocks(StatementBlock.mergeFunctionCalls(dmlp.getStatementBlocks(), dmlp)); - for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) { + for(int i = 0; i < dmlp.getNumStatementBlocks(); i++) { StatementBlock sb = dmlp.getStatementBlock(i); activeIn = sb.initializeforwardLV(activeIn); } - if (dmlp.getNumStatementBlocks() > 0){ + if(dmlp.getNumStatementBlocks() > 0) { StatementBlock lastSb = dmlp.getStatementBlock(dmlp.getNumStatementBlocks() - 1); lastSb._liveOut = new VariableSet(); - for (int i = dmlp.getNumStatementBlocks() - 1; i >= 0; i--) { + for(int i = dmlp.getNumStatementBlocks() - 1; i >= 0; i--) { StatementBlock sb = dmlp.getStatementBlock(i); currentLiveOut = sb.analyze(currentLiveOut); } @@ -213,14 +210,14 @@ public void liveVariableAnalysis(DMLProgram dmlp, boolean inclFuns) { public void liveVariableAnalysisFunction(DMLProgram dmlp, FunctionStatementBlock fsb) { //STEP 1: forward direction - FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0); + FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0); // perform function inlining fstmt.setBody(StatementBlock.mergeFunctionCalls(fstmt.getBody(), dmlp)); VariableSet activeIn = new VariableSet(); - for (DataIdentifier id : fstmt.getInputParams()){ - activeIn.addVariable(id.getName(), id); + for(DataIdentifier id : fstmt.getInputParams()) { + activeIn.addVariable(id.getName(), id); } fsb.initializeforwardLV(activeIn); @@ -229,10 +226,10 @@ public void liveVariableAnalysisFunction(DMLProgram dmlp, FunctionStatementBlock VariableSet currentLiveIn = new VariableSet(); VariableSet unionLiveIn = new VariableSet(); - for (DataIdentifier id : fstmt.getInputParams()) + for(DataIdentifier id : fstmt.getInputParams()) currentLiveIn.addVariable(id.getName(), id); - for (DataIdentifier id : fstmt.getOutputParams()) { + for(DataIdentifier id : fstmt.getOutputParams()) { currentLiveOut.addVariable(id.getName(), id); unionLiveIn.addVariable(id.getName(), id); } @@ -245,11 +242,10 @@ public void liveVariableAnalysisFunction(DMLProgram dmlp, FunctionStatementBlock public void cleanupLiveOutVariables(List sbs, VariableSet unionLiveIn) { //backwards pass to collect union of livein variables of all successors //and cleanup unnecessary liveout variables - for(int i=sbs.size()-1; i>=0; i--) { + for(int i = sbs.size() - 1; i >= 0; i--) { StatementBlock sb = sbs.get(i); //remove liveout variables that are not in unionLivein - sb.liveOut().removeVariables( - VariableSet.minus(sb.liveOut(), unionLiveIn)); + sb.liveOut().removeVariables(VariableSet.minus(sb.liveOut(), unionLiveIn)); //collect all livein information unionLiveIn.addVariables(sb.liveIn()); } @@ -261,23 +257,22 @@ public void constructHops(DMLProgram dmlp) { public void constructHops(DMLProgram dmlp, boolean inclFuns) { // Step 1: construct hops for all functions - if( inclFuns ) { + if(inclFuns) { // for each namespace, handle function program blocks - for( FunctionDictionary fdict : dmlp.getNamespaces().values() ) - for( FunctionStatementBlock fsb : fdict.getFunctions().values() ) + for(FunctionDictionary fdict : dmlp.getNamespaces().values()) + for(FunctionStatementBlock fsb : fdict.getFunctions().values()) constructHops(fsb); } // Step 2: construct hops for main program // handle regular program blocks - for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) { + for(int i = 0; i < dmlp.getNumStatementBlocks(); i++) { StatementBlock current = dmlp.getStatementBlock(i); constructHops(current); } } - public void rewriteHopsDAG(DMLProgram dmlp) - { + public void rewriteHopsDAG(DMLProgram dmlp) { //apply hop rewrites (static rewrites) ProgramRewriter rewriter = new ProgramRewriter(true, false); rewriter.rewriteProgramHopDAGs(dmlp, false); //rewrite and merge @@ -286,7 +281,7 @@ public void rewriteHopsDAG(DMLProgram dmlp) resetHopsDAGVisitStatus(dmlp); //propagate size information from main into functions (but conservatively) - if( OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS ) { + if(OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS) { InterProceduralAnalysis ipa = new InterProceduralAnalysis(dmlp); ipa.analyzeProgram(OptimizerUtils.IPA_NUM_REPETITIONS); resetHopsDAGVisitStatus(dmlp); @@ -304,13 +299,12 @@ public void rewriteHopsDAG(DMLProgram dmlp) //enhance HOP DAGs by automatic operator fusion DMLConfig dmlconf = ConfigurationManager.getDMLConfig(); - if( ConfigurationManager.isCodegenEnabled() ){ - SpoofCompiler.PLAN_CACHE_POLICY = PlanCachePolicy.get( - dmlconf.getBooleanValue(DMLConfig.CODEGEN_PLANCACHE), - dmlconf.getIntValue(DMLConfig.CODEGEN_LITERALS)==2); + if(ConfigurationManager.isCodegenEnabled()) { + SpoofCompiler.PLAN_CACHE_POLICY = PlanCachePolicy.get(dmlconf.getBooleanValue(DMLConfig.CODEGEN_PLANCACHE), + dmlconf.getIntValue(DMLConfig.CODEGEN_LITERALS) == 2); SpoofCompiler.setConfiguredPlanSelector(); SpoofCompiler.setExecTypeSpecificJavaCompiler(); - if( SpoofCompiler.INTEGRATION==IntegrationType.HOPS ) + if(SpoofCompiler.INTEGRATION == IntegrationType.HOPS) codgenHopsDAG(dmlp); } } @@ -334,33 +328,31 @@ public void codgenHopsDAG(ProgramBlock pb) { public void constructLops(DMLProgram dmlp) { // for each namespace, handle function program blocks - for( FunctionDictionary fdict : dmlp.getNamespaces().values() ) { + for(FunctionDictionary fdict : dmlp.getNamespaces().values()) { //handle optimized functions - for( FunctionStatementBlock fsb : fdict.getFunctions().values() ) + for(FunctionStatementBlock fsb : fdict.getFunctions().values()) constructLops(fsb); //handle unoptimized functions - if( fdict.getFunctions(false) != null ) - for( FunctionStatementBlock fsb : fdict.getFunctions(false).values() ) + if(fdict.getFunctions(false) != null) + for(FunctionStatementBlock fsb : fdict.getFunctions(false).values()) constructLops(fsb); } // handle regular program blocks - for( StatementBlock sb : dmlp.getStatementBlocks() ) + for(StatementBlock sb : dmlp.getStatementBlocks()) constructLops(sb); } - public boolean constructLops(StatementBlock sb) - { + public boolean constructLops(StatementBlock sb) { boolean ret = false; - if (sb instanceof WhileStatementBlock) - { - WhileStatementBlock wsb = (WhileStatementBlock)sb; - WhileStatement whileStmt = (WhileStatement)wsb.getStatement(0); + if(sb instanceof WhileStatementBlock) { + WhileStatementBlock wsb = (WhileStatementBlock) sb; + WhileStatement whileStmt = (WhileStatement) wsb.getStatement(0); ArrayList body = whileStmt.getBody(); // step through stmt blocks in while stmt body - for (StatementBlock stmtBlock : body) + for(StatementBlock stmtBlock : body) ret |= constructLops(stmtBlock); // handle while stmt predicate @@ -369,19 +361,18 @@ public boolean constructLops(StatementBlock sb) ret |= wsb.updatePredicateRecompilationFlag(); } - else if (sb instanceof IfStatementBlock) - { + else if(sb instanceof IfStatementBlock) { IfStatementBlock isb = (IfStatementBlock) sb; - IfStatement ifStmt = (IfStatement)isb.getStatement(0); + IfStatement ifStmt = (IfStatement) isb.getStatement(0); ArrayList ifBody = ifStmt.getIfBody(); ArrayList elseBody = ifStmt.getElseBody(); // step through stmt blocks in if stmt ifBody - for (StatementBlock stmtBlock : ifBody) + for(StatementBlock stmtBlock : ifBody) ret |= constructLops(stmtBlock); // step through stmt blocks in if stmt elseBody - for (StatementBlock stmtBlock : elseBody) + for(StatementBlock stmtBlock : elseBody) ret |= constructLops(stmtBlock); // handle if stmt predicate @@ -390,49 +381,49 @@ else if (sb instanceof IfStatementBlock) ret |= isb.updatePredicateRecompilationFlag(); } - else if (sb instanceof ForStatementBlock) //NOTE: applies to ForStatementBlock and ParForStatementBlock + else if(sb instanceof ForStatementBlock) //NOTE: applies to ForStatementBlock and ParForStatementBlock { - ForStatementBlock fsb = (ForStatementBlock) sb; - ForStatement fs = (ForStatement)sb.getStatement(0); + ForStatementBlock fsb = (ForStatementBlock) sb; + ForStatement fs = (ForStatement) sb.getStatement(0); ArrayList body = fs.getBody(); // step through stmt blocks in FOR stmt body - for (StatementBlock stmtBlock : body) + for(StatementBlock stmtBlock : body) ret |= constructLops(stmtBlock); // handle for stmt predicate - if (fsb.getFromHops() != null){ + if(fsb.getFromHops() != null) { Lop llobs = fsb.getFromHops().constructLops(); fsb.setFromLops(llobs); } - if (fsb.getToHops() != null){ + if(fsb.getToHops() != null) { Lop llobs = fsb.getToHops().constructLops(); fsb.setToLops(llobs); } - if (fsb.getIncrementHops() != null){ + if(fsb.getIncrementHops() != null) { Lop llobs = fsb.getIncrementHops().constructLops(); fsb.setIncrementLops(llobs); } ret |= fsb.updatePredicateRecompilationFlags(); } - else if (sb instanceof FunctionStatementBlock) { + else if(sb instanceof FunctionStatementBlock) { FunctionStatementBlock fsb = (FunctionStatementBlock) sb; - FunctionStatement functStmt = (FunctionStatement)sb.getStatement(0); + FunctionStatement functStmt = (FunctionStatement) sb.getStatement(0); ArrayList body = functStmt.getBody(); // step through stmt blocks in while stmt body - for( StatementBlock stmtBlock : body ) + for(StatementBlock stmtBlock : body) ret |= constructLops(stmtBlock); - if( fsb.isRecompileOnce() ) + if(fsb.isRecompileOnce()) fsb.setRecompileOnce(ret); } // handle default case for regular StatementBlock else { - if (sb.getHops() == null) + if(sb.getHops() == null) sb.setHops(new ArrayList()); ArrayList lops = new ArrayList<>(); - for (Hop hop : sb.getHops()) + for(Hop hop : sb.getHops()) lops.add(hop.constructLops()); sb.setLops(lops); ret |= sb.updateRecompilationFlag(); @@ -441,21 +432,20 @@ else if (sb instanceof FunctionStatementBlock) { return ret; } - public Program getRuntimeProgram(DMLProgram prog, DMLConfig config) - throws LanguageException, DMLRuntimeException, LopsException, HopsException - { + public Program getRuntimeProgram(DMLProgram prog, DMLConfig config) + throws LanguageException, DMLRuntimeException, LopsException, HopsException { // constructor resets the set of registered functions Program rtprog = new Program(prog); // for all namespaces, translate function statement blocks into function program blocks - for (String namespace : prog.getNamespaces().keySet()){ + for(String namespace : prog.getNamespaces().keySet()) { - for (String fname : prog.getFunctionStatementBlocks(namespace).keySet()){ + for(String fname : prog.getFunctionStatementBlocks(namespace).keySet()) { // add program block to program FunctionStatementBlock fsb = prog.getFunctionStatementBlocks(namespace).get(fname); prepareAndAddFunctionProgramBlock(rtprog, config, namespace, fname, fsb, true); // add unoptimized block to program (for second-order calls) - if( prog.getNamespaces().get(namespace).containsFunction(fname, false) ) { + if(prog.getNamespaces().get(namespace).containsFunction(fname, false)) { prepareAndAddFunctionProgramBlock(rtprog, config, namespace, fname, prog.getNamespaces().get(namespace).getFunction(fname, false), false); } @@ -463,25 +453,23 @@ public Program getRuntimeProgram(DMLProgram prog, DMLConfig config) } // translate all top-level statement blocks to program blocks - for (StatementBlock sb : prog.getStatementBlocks() ) { + for(StatementBlock sb : prog.getStatementBlocks()) { // add program block to program ProgramBlock rtpb = createRuntimeProgramBlock(rtprog, sb, config); rtprog.addProgramBlock(rtpb); } //enhance runtime program by automatic operator fusion - if( ConfigurationManager.isCodegenEnabled() - && SpoofCompiler.INTEGRATION==IntegrationType.RUNTIME ){ + if(ConfigurationManager.isCodegenEnabled() && SpoofCompiler.INTEGRATION == IntegrationType.RUNTIME) { codgenHopsDAG(rtprog); } - return rtprog ; + return rtprog; } - private void prepareAndAddFunctionProgramBlock(Program rtprog, DMLConfig config, - String fnamespace, String fname, FunctionStatementBlock fsb, boolean opt) - { - FunctionProgramBlock rtpb = (FunctionProgramBlock)createRuntimeProgramBlock(rtprog, fsb, config); + private void prepareAndAddFunctionProgramBlock(Program rtprog, DMLConfig config, String fnamespace, String fname, + FunctionStatementBlock fsb, boolean opt) { + FunctionProgramBlock rtpb = (FunctionProgramBlock) createRuntimeProgramBlock(rtprog, fsb, config); rtprog.addFunctionProgramBlock(fnamespace, fname, rtpb, opt); rtpb.setRecompileOnce(fsb.isRecompileOnce()); rtpb.setNondeterministic(fsb.isNondeterministic()); @@ -497,7 +485,7 @@ public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, D ProgramBlock retPB = null; // process While Statement - add runtime program blocks to program - if (sb instanceof WhileStatementBlock){ + if(sb instanceof WhileStatementBlock) { // create DAG for loop predicates pred_dag = new Dag<>(); @@ -506,7 +494,7 @@ public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, D // create instructions for loop predicates pred_instruct = new ArrayList<>(); ArrayList pInst = pred_dag.getJobs(null, config); - for (Instruction i : pInst ) { + for(Instruction i : pInst) { pred_instruct.add(i); } @@ -515,9 +503,9 @@ public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, D //// process the body of the while statement block //// - WhileStatementBlock wsb = (WhileStatementBlock)sb; - WhileStatement wstmt = (WhileStatement)wsb.getStatement(0); - for (StatementBlock sblock : wstmt.getBody()){ + WhileStatementBlock wsb = (WhileStatementBlock) sb; + WhileStatement wstmt = (WhileStatement) wsb.getStatement(0); + for(StatementBlock sblock : wstmt.getBody()) { // process the body ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config); @@ -537,7 +525,7 @@ public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, D } // process If Statement - add runtime program blocks to program - else if (sb instanceof IfStatementBlock){ + else if(sb instanceof IfStatementBlock) { // create DAG for loop predicates pred_dag = new Dag<>(); @@ -546,7 +534,7 @@ else if (sb instanceof IfStatementBlock){ // create instructions for loop predicates pred_instruct = new ArrayList<>(); ArrayList pInst = pred_dag.getJobs(null, config); - for (Instruction i : pInst ) { + for(Instruction i : pInst) { pred_instruct.add(i); } @@ -554,19 +542,19 @@ else if (sb instanceof IfStatementBlock){ IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct); // process the body of the if statement block - IfStatementBlock isb = (IfStatementBlock)sb; - IfStatement istmt = (IfStatement)isb.getStatement(0); + IfStatementBlock isb = (IfStatementBlock) sb; + IfStatement istmt = (IfStatement) isb.getStatement(0); // process the if body - for (StatementBlock sblock : istmt.getIfBody()){ + for(StatementBlock sblock : istmt.getIfBody()) { ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config); rtpb.addProgramBlockIfBody(childBlock); } // process the else body - for (StatementBlock sblock : istmt.getElseBody()){ + for(StatementBlock sblock : istmt.getElseBody()) { ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config); - rtpb.addProgramBlockElseBody(childBlock); + rtpb.addProgramBlockElseBody(childBlock); } retPB = rtpb; @@ -583,19 +571,18 @@ else if (sb instanceof IfStatementBlock){ // process For Statement - add runtime program blocks to program // NOTE: applies to ForStatementBlock and ParForStatementBlock - else if (sb instanceof ForStatementBlock) - { + else if(sb instanceof ForStatementBlock) { ForStatementBlock fsb = (ForStatementBlock) sb; // create DAGs for loop predicates Dag fromDag = new Dag<>(); Dag toDag = new Dag<>(); Dag incrementDag = new Dag<>(); - if( fsb.getFromHops()!=null ) + if(fsb.getFromHops() != null) fsb.getFromLops().addToDag(fromDag); - if( fsb.getToHops()!=null ) + if(fsb.getToHops() != null) fsb.getToLops().addToDag(toDag); - if( fsb.getIncrementHops()!=null ) + if(fsb.getIncrementHops() != null) fsb.getIncrementLops().addToDag(incrementDag); // create instructions for loop predicates @@ -607,10 +594,10 @@ else if (sb instanceof ForStatementBlock) ForProgramBlock rtpb = null; IterablePredicate iterPred = fsb.getIterPredicate(); - if( sb instanceof ParForStatementBlock && ConfigurationManager.isParallelParFor() ) { - rtpb = new ParForProgramBlock(prog, iterPred.getIterVar().getName(), - iterPred.getParForParams(), ((ParForStatementBlock)sb).getResultVariables()); - ParForProgramBlock pfrtpb = (ParForProgramBlock)rtpb; + if(sb instanceof ParForStatementBlock && ConfigurationManager.isParallelParFor()) { + rtpb = new ParForProgramBlock(prog, iterPred.getIterVar().getName(), iterPred.getParForParams(), + ((ParForStatementBlock) sb).getResultVariables()); + ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb; pfrtpb.setStatementBlock(sb); //used for optimization and creating unscoped variables } else {//ForStatementBlock @@ -622,10 +609,10 @@ else if (sb instanceof ForStatementBlock) rtpb.setIncrementInstructions(incrementInstructions); // process the body of the for statement block - ForStatement fs = (ForStatement)fsb.getStatement(0); - for (StatementBlock sblock : fs.getBody()){ + ForStatement fs = (ForStatement) fsb.getStatement(0); + for(StatementBlock sblock : fs.getBody()) { ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config); - rtpb.addProgramBlock(childBlock); + rtpb.addProgramBlock(childBlock); } retPB = rtpb; @@ -641,24 +628,24 @@ else if (sb instanceof ForStatementBlock) } // process function statement block - add runtime program blocks to program - else if (sb instanceof FunctionStatementBlock){ + else if(sb instanceof FunctionStatementBlock) { - FunctionStatementBlock fsb = (FunctionStatementBlock)sb; - FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0); + FunctionStatementBlock fsb = (FunctionStatementBlock) sb; + FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0); FunctionProgramBlock rtpb = null; // create function program block rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams()); // process the function statement body - for (StatementBlock sblock : fstmt.getBody()){ + for(StatementBlock sblock : fstmt.getBody()) { // process the body ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config); rtpb.addProgramBlock(childBlock); } // check there are actually Lops in to process (loop stmt body will not have any) - if (fsb.getLops() != null && !fsb.getLops().isEmpty()){ + if(fsb.getLops() != null && !fsb.getLops().isEmpty()) { throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops"); } @@ -679,9 +666,9 @@ else if (sb instanceof FunctionStatementBlock){ dag = new Dag<>(); // check there are actually Lops in to process (loop stmt body will not have any) - if (sb.getLops() != null && !sb.getLops().isEmpty()){ + if(sb.getLops() != null && !sb.getLops().isEmpty()) { - for (Lop l : sb.getLops()) { + for(Lop l : sb.getLops()) { l.addToDag(dag); } @@ -708,90 +695,88 @@ else if (sb instanceof FunctionStatementBlock){ public static void refreshMemEstimates(DMLProgram dmlp) { // for each namespace, handle function program blocks -- forward direction - for (String namespaceKey : dmlp.getNamespaces().keySet()){ - for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){ + for(String namespaceKey : dmlp.getNamespaces().keySet()) { + for(String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) { FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname); refreshMemEstimates(fsblock); } } // handle statement blocks in "main" method - for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) { + for(int i = 0; i < dmlp.getNumStatementBlocks(); i++) { StatementBlock current = dmlp.getStatementBlock(i); refreshMemEstimates(current); } } private static Instruction deriveExitInstruction(StatementBlock sb) { - Set rmVars = VariableSet.union( - VariableSet.minus(sb.liveIn(), sb.liveOut()), + Set rmVars = VariableSet.union(VariableSet.minus(sb.liveIn(), sb.liveOut()), VariableSet.minus(sb.getKill(), sb.liveOut())).getVariableNames(); - return rmVars.isEmpty() ? null : - VariableCPInstruction.prepareRemoveInstruction(rmVars.toArray(new String[0])); + return rmVars.isEmpty() ? null : VariableCPInstruction.prepareRemoveInstruction(rmVars.toArray(new String[0])); } public static void refreshMemEstimates(StatementBlock current) { MemoTable memo = new MemoTable(); - if( HopRewriteUtils.isLastLevelStatementBlock(current) ) { + if(HopRewriteUtils.isLastLevelStatementBlock(current)) { ArrayList hopsDAG = current.getHops(); - if (hopsDAG != null && !hopsDAG.isEmpty()) - for( Hop hop : hopsDAG ) + if(hopsDAG != null && !hopsDAG.isEmpty()) + for(Hop hop : hopsDAG) hop.refreshMemEstimates(memo); } - if (current instanceof FunctionStatementBlock) { + if(current instanceof FunctionStatementBlock) { - FunctionStatement fstmt = (FunctionStatement)current.getStatement(0); - for (StatementBlock sb : fstmt.getBody()){ + FunctionStatement fstmt = (FunctionStatement) current.getStatement(0); + for(StatementBlock sb : fstmt.getBody()) { refreshMemEstimates(sb); } } - else if (current instanceof WhileStatementBlock) { + else if(current instanceof WhileStatementBlock) { // handle predicate WhileStatementBlock wstb = (WhileStatementBlock) current; wstb.getPredicateHops().refreshMemEstimates(new MemoTable()); - if (wstb.getNumStatements() > 1) + if(wstb.getNumStatements() > 1) LOG.debug("While statement block has more than 1 stmt"); - WhileStatement ws = (WhileStatement)wstb.getStatement(0); + WhileStatement ws = (WhileStatement) wstb.getStatement(0); - for (StatementBlock sb : ws.getBody()){ + for(StatementBlock sb : ws.getBody()) { refreshMemEstimates(sb); } } - else if (current instanceof IfStatementBlock) { + else if(current instanceof IfStatementBlock) { // handle predicate IfStatementBlock istb = (IfStatementBlock) current; istb.getPredicateHops().refreshMemEstimates(new MemoTable()); - if (istb.getNumStatements() > 1) + if(istb.getNumStatements() > 1) LOG.debug("If statement block has more than 1 stmt"); - IfStatement is = (IfStatement)istb.getStatement(0); + IfStatement is = (IfStatement) istb.getStatement(0); - for (StatementBlock sb : is.getIfBody()){ + for(StatementBlock sb : is.getIfBody()) { refreshMemEstimates(sb); } - for (StatementBlock sb : is.getElseBody()){ + for(StatementBlock sb : is.getElseBody()) { refreshMemEstimates(sb); } } - else if (current instanceof ForStatementBlock) { + else if(current instanceof ForStatementBlock) { // handle predicate ForStatementBlock fsb = (ForStatementBlock) current; - if (fsb.getFromHops() != null) + if(fsb.getFromHops() != null) fsb.getFromHops().refreshMemEstimates(new MemoTable()); - if (fsb.getToHops() != null) + if(fsb.getToHops() != null) fsb.getToHops().refreshMemEstimates(new MemoTable()); - if (fsb.getIncrementHops() != null) + if(fsb.getIncrementHops() != null) fsb.getIncrementHops().refreshMemEstimates(new MemoTable()); - if (fsb.getNumStatements() > 1) + if(fsb.getNumStatements() > 1) LOG.debug("For statement block has more than 1 stmt"); - ForStatement ws = (ForStatement)fsb.getStatement(0); + ForStatement ws = (ForStatement) fsb.getStatement(0); - for (StatementBlock sb : ws.getBody()){ + for(StatementBlock sb : ws.getBody()) { refreshMemEstimates(sb); } } @@ -800,15 +785,15 @@ else if (current instanceof ForStatementBlock) { public static void resetHopsDAGVisitStatus(DMLProgram dmlp) { // for each namespace, handle function program blocks -- forward direction - for (String namespaceKey : dmlp.getNamespaces().keySet()){ - for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){ + for(String namespaceKey : dmlp.getNamespaces().keySet()) { + for(String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) { FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname); resetHopsDAGVisitStatus(fsblock); } } // handle statement blocks in "main" method - for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) { + for(int i = 0; i < dmlp.getNumStatementBlocks(); i++) { StatementBlock current = dmlp.getStatementBlock(i); resetHopsDAGVisitStatus(current); } @@ -816,54 +801,54 @@ public static void resetHopsDAGVisitStatus(DMLProgram dmlp) { public static void resetHopsDAGVisitStatus(StatementBlock current) { - if( HopRewriteUtils.isLastLevelStatementBlock(current) ) { + if(HopRewriteUtils.isLastLevelStatementBlock(current)) { ArrayList hopsDAG = current.getHops(); - if (hopsDAG != null && !hopsDAG.isEmpty() ) { + if(hopsDAG != null && !hopsDAG.isEmpty()) { Hop.resetVisitStatus(hopsDAG); } } - if (current instanceof FunctionStatementBlock) { - FunctionStatement fstmt = (FunctionStatement)current.getStatement(0); - for (StatementBlock sb : fstmt.getBody()){ + if(current instanceof FunctionStatementBlock) { + FunctionStatement fstmt = (FunctionStatement) current.getStatement(0); + for(StatementBlock sb : fstmt.getBody()) { resetHopsDAGVisitStatus(sb); } } - else if (current instanceof WhileStatementBlock) { + else if(current instanceof WhileStatementBlock) { // handle predicate WhileStatementBlock wstb = (WhileStatementBlock) current; wstb.getPredicateHops().resetVisitStatus(); - WhileStatement ws = (WhileStatement)wstb.getStatement(0); - for (StatementBlock sb : ws.getBody()) + WhileStatement ws = (WhileStatement) wstb.getStatement(0); + for(StatementBlock sb : ws.getBody()) resetHopsDAGVisitStatus(sb); } - else if (current instanceof IfStatementBlock) { + else if(current instanceof IfStatementBlock) { // handle predicate IfStatementBlock istb = (IfStatementBlock) current; istb.getPredicateHops().resetVisitStatus(); - IfStatement is = (IfStatement)istb.getStatement(0); - for (StatementBlock sb : is.getIfBody()) + IfStatement is = (IfStatement) istb.getStatement(0); + for(StatementBlock sb : is.getIfBody()) resetHopsDAGVisitStatus(sb); - for (StatementBlock sb : is.getElseBody()) + for(StatementBlock sb : is.getElseBody()) resetHopsDAGVisitStatus(sb); } - else if (current instanceof ForStatementBlock) { + else if(current instanceof ForStatementBlock) { // handle predicate ForStatementBlock fsb = (ForStatementBlock) current; - if (fsb.getFromHops() != null) + if(fsb.getFromHops() != null) fsb.getFromHops().resetVisitStatus(); - if (fsb.getToHops() != null) + if(fsb.getToHops() != null) fsb.getToHops().resetVisitStatus(); - if (fsb.getIncrementHops() != null) + if(fsb.getIncrementHops() != null) fsb.getIncrementHops().resetVisitStatus(); - if (fsb.getNumStatements() > 1) + if(fsb.getNumStatements() > 1) LOG.debug("For statment block has more than 1 stmt"); - ForStatement ws = (ForStatement)fsb.getStatement(0); + ForStatement ws = (ForStatement) fsb.getStatement(0); - for (StatementBlock sb : ws.getBody()){ + for(StatementBlock sb : ws.getBody()) { resetHopsDAGVisitStatus(sb); } } @@ -872,14 +857,14 @@ else if (current instanceof ForStatementBlock) { public void resetLopsDAGVisitStatus(DMLProgram dmlp) { // for each namespace, handle function program blocks - for (String namespaceKey : dmlp.getNamespaces().keySet()){ - for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){ + for(String namespaceKey : dmlp.getNamespaces().keySet()) { + for(String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) { FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname); resetLopsDAGVisitStatus(fsblock); } } - for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) { + for(int i = 0; i < dmlp.getNumStatementBlocks(); i++) { StatementBlock current = dmlp.getStatementBlock(i); resetLopsDAGVisitStatus(current); } @@ -889,88 +874,88 @@ public void resetLopsDAGVisitStatus(StatementBlock current) { ArrayList hopsDAG = current.getHops(); - if (hopsDAG != null && !hopsDAG.isEmpty() ) { + if(hopsDAG != null && !hopsDAG.isEmpty()) { Iterator iter = hopsDAG.iterator(); - while (iter.hasNext()){ + while(iter.hasNext()) { Hop currentHop = iter.next(); currentHop.getLops().resetVisitStatus(); } } - if (current instanceof FunctionStatementBlock) { + if(current instanceof FunctionStatementBlock) { FunctionStatementBlock fsb = (FunctionStatementBlock) current; - FunctionStatement fs = (FunctionStatement)fsb.getStatement(0); + FunctionStatement fs = (FunctionStatement) fsb.getStatement(0); - for (StatementBlock sb : fs.getBody()){ + for(StatementBlock sb : fs.getBody()) { resetLopsDAGVisitStatus(sb); } } - if (current instanceof WhileStatementBlock) { + if(current instanceof WhileStatementBlock) { WhileStatementBlock wstb = (WhileStatementBlock) current; wstb.getPredicateLops().resetVisitStatus(); - if (wstb.getNumStatements() > 1) + if(wstb.getNumStatements() > 1) LOG.debug("While statement block has more than 1 stmt"); - WhileStatement ws = (WhileStatement)wstb.getStatement(0); + WhileStatement ws = (WhileStatement) wstb.getStatement(0); - for (StatementBlock sb : ws.getBody()){ + for(StatementBlock sb : ws.getBody()) { resetLopsDAGVisitStatus(sb); } } - if (current instanceof IfStatementBlock) { + if(current instanceof IfStatementBlock) { IfStatementBlock istb = (IfStatementBlock) current; istb.getPredicateLops().resetVisitStatus(); - if (istb.getNumStatements() > 1) + if(istb.getNumStatements() > 1) LOG.debug("If statement block has more than 1 stmt"); - IfStatement is = (IfStatement)istb.getStatement(0); + IfStatement is = (IfStatement) istb.getStatement(0); - for (StatementBlock sb : is.getIfBody()){ + for(StatementBlock sb : is.getIfBody()) { resetLopsDAGVisitStatus(sb); } - for (StatementBlock sb : is.getElseBody()){ + for(StatementBlock sb : is.getElseBody()) { resetLopsDAGVisitStatus(sb); } } - if (current instanceof ForStatementBlock) { + if(current instanceof ForStatementBlock) { ForStatementBlock fsb = (ForStatementBlock) current; - if (fsb.getFromLops() != null) + if(fsb.getFromLops() != null) fsb.getFromLops().resetVisitStatus(); - if (fsb.getToLops() != null) + if(fsb.getToLops() != null) fsb.getToLops().resetVisitStatus(); - if (fsb.getIncrementLops() != null) + if(fsb.getIncrementLops() != null) fsb.getIncrementLops().resetVisitStatus(); - if (fsb.getNumStatements() > 1) + if(fsb.getNumStatements() > 1) LOG.debug("For statement block has more than 1 stmt"); - ForStatement ws = (ForStatement)fsb.getStatement(0); + ForStatement ws = (ForStatement) fsb.getStatement(0); - for (StatementBlock sb : ws.getBody()){ + for(StatementBlock sb : ws.getBody()) { resetLopsDAGVisitStatus(sb); } } } public void constructHops(StatementBlock sb) { - if (sb instanceof WhileStatementBlock) { + if(sb instanceof WhileStatementBlock) { constructHopsForWhileControlBlock((WhileStatementBlock) sb); return; } - if (sb instanceof IfStatementBlock) { + if(sb instanceof IfStatementBlock) { constructHopsForIfControlBlock((IfStatementBlock) sb); return; } - if (sb instanceof ForStatementBlock) { //incl ParForStatementBlock + if(sb instanceof ForStatementBlock) { //incl ParForStatementBlock constructHopsForForControlBlock((ForStatementBlock) sb); return; } - if (sb instanceof FunctionStatementBlock) { + if(sb instanceof FunctionStatementBlock) { constructHopsForFunctionControlBlock((FunctionStatementBlock) sb); return; } @@ -978,31 +963,31 @@ public void constructHops(StatementBlock sb) { HashMap ids = new HashMap<>(); ArrayList output = new ArrayList<>(); - VariableSet liveIn = sb.liveIn(); + VariableSet liveIn = sb.liveIn(); VariableSet liveOut = sb.liveOut(); VariableSet updated = sb._updated; - VariableSet gen = sb._gen; + VariableSet gen = sb._gen; VariableSet updatedLiveOut = new VariableSet(); // handle liveout variables that are updated --> target identifiers for Assignment HashMap liveOutToTemp = new HashMap<>(); - for (int i = 0; i < sb.getNumStatements(); i++) { + for(int i = 0; i < sb.getNumStatements(); i++) { Statement current = sb.getStatement(i); - if (current instanceof AssignmentStatement) { + if(current instanceof AssignmentStatement) { AssignmentStatement as = (AssignmentStatement) current; DataIdentifier target = as.getTarget(); - if (target != null) { - if (liveOut.containsVariable(target.getName())) { + if(target != null) { + if(liveOut.containsVariable(target.getName())) { liveOutToTemp.put(target.getName(), Integer.valueOf(i)); } } } - if (current instanceof MultiAssignmentStatement) { + if(current instanceof MultiAssignmentStatement) { MultiAssignmentStatement mas = (MultiAssignmentStatement) current; - for (DataIdentifier target : mas.getTargetList()){ - if (liveOut.containsVariable(target.getName())) { + for(DataIdentifier target : mas.getTargetList()) { + if(liveOut.containsVariable(target.getName())) { liveOutToTemp.put(target.getName(), Integer.valueOf(i)); } } @@ -1011,43 +996,45 @@ public void constructHops(StatementBlock sb) { // only create transient read operations for variables either updated or read-before-update // (i.e., from LV analysis, updated and gen sets) - if ( !liveIn.getVariables().values().isEmpty() ) { + if(!liveIn.getVariables().values().isEmpty()) { - for (String varName : liveIn.getVariables().keySet()) { + for(String varName : liveIn.getVariables().keySet()) { - if (updated.containsVariable(varName) || gen.containsVariable(varName)){ + if(updated.containsVariable(varName) || gen.containsVariable(varName)) { DataIdentifier var = liveIn.getVariables().get(varName); - long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1(); - long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2(); - DataOp read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), OpOpData.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getBlocksize()); + long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim1() : var.getDim1(); + long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim2() : var.getDim2(); + DataOp read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), + OpOpData.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getBlocksize()); read.setParseInfo(var); ids.put(varName, read); } } } - for( int i = 0; i < sb.getNumStatements(); i++ ) { + for(int i = 0; i < sb.getNumStatements(); i++) { Statement current = sb.getStatement(i); - if (current instanceof OutputStatement) { + if(current instanceof OutputStatement) { OutputStatement os = (OutputStatement) current; DataExpression source = os.getSource(); DataIdentifier target = os.getIdentifier(); //error handling unsupported indexing expression in write statement - if( target instanceof IndexedIdentifier ) { - throw new LanguageException(source.printErrorLocation()+": Unsupported indexing expression in write statement. " + - "Please, assign the right indexing result to a variable and write this variable."); + if(target instanceof IndexedIdentifier) { + throw new LanguageException( + source.printErrorLocation() + ": Unsupported indexing expression in write statement. " + + "Please, assign the right indexing result to a variable and write this variable."); } - DataOp ae = (DataOp)processExpression(source, target, ids); + DataOp ae = (DataOp) processExpression(source, target, ids); Expression fmtExpr = os.getExprParam(DataExpression.FORMAT_TYPE); - ae.setFileFormat((fmtExpr instanceof StringIdentifier) ? - Expression.convertFormatType(fmtExpr.toString()) : FileFormat.UNKNOWN); + ae.setFileFormat((fmtExpr instanceof StringIdentifier) ? Expression.convertFormatType( + fmtExpr.toString()) : FileFormat.UNKNOWN); - if (ae.getDataType() == DataType.SCALAR ) { + if(ae.getDataType() == DataType.SCALAR) { ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1); } else { @@ -1064,7 +1051,8 @@ public void constructHops(StatementBlock sb) { case COMPRESSED: case UNKNOWN: // write output in binary block format - ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), ae.getBlocksize()); + ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), + ae.getBlocksize()); break; case FEDERATED: ae.setOutputParams(ae.getDim1(), ae.getDim2(), -1, ae.getUpdateType(), -1); @@ -1077,7 +1065,7 @@ public void constructHops(StatementBlock sb) { output.add(ae); } - if (current instanceof PrintStatement) { + if(current instanceof PrintStatement) { DataIdentifier target = createTarget(); target.setDataType(DataType.SCALAR); target.setValueType(ValueType.STRING); @@ -1087,66 +1075,71 @@ public void constructHops(StatementBlock sb) { PRINTTYPE ptype = ps.getType(); try { - if (ptype == PRINTTYPE.PRINT) { + if(ptype == PRINTTYPE.PRINT) { OpOp1 op = OpOp1.PRINT; Expression source = ps.getExpressions().get(0); Hop ae = processExpression(source, target, ids); - Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae); + Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, + ae); printHop.setParseInfo(current); output.add(printHop); } - else if (ptype == PRINTTYPE.ASSERT) { + else if(ptype == PRINTTYPE.ASSERT) { OpOp1 op = OpOp1.ASSERT; Expression source = ps.getExpressions().get(0); Hop ae = processExpression(source, target, ids); - Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae); + Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, + ae); printHop.setParseInfo(current); output.add(printHop); } - else if (ptype == PRINTTYPE.STOP) { + else if(ptype == PRINTTYPE.STOP) { OpOp1 op = OpOp1.STOP; Expression source = ps.getExpressions().get(0); Hop ae = processExpression(source, target, ids); - Hop stopHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae); + Hop stopHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, + ae); stopHop.setParseInfo(current); output.add(stopHop); sb.setSplitDag(true); //avoid merge - } else if (ptype == PRINTTYPE.PRINTF) { + } + else if(ptype == PRINTTYPE.PRINTF) { List expressions = ps.getExpressions(); Hop[] inHops = new Hop[expressions.size()]; // process the expressions (function parameters) that // make up the printf-styled print statement // into Hops so that these can be passed to the printf // Hop (ie, MultipleOp) as input Hops - for (int j = 0; j < expressions.size(); j++) { + for(int j = 0; j < expressions.size(); j++) { Hop inHop = processExpression(expressions.get(j), target, ids); inHops[j] = inHop; } target.setValueType(ValueType.STRING); - Hop printfHop = new NaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOpN.PRINTF, inHops); + Hop printfHop = new NaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOpN.PRINTF, inHops); output.add(printfHop); } - } catch (HopsException e) { + } + catch(HopsException e) { throw new LanguageException(e); } } - if (current instanceof AssignmentStatement) { + if(current instanceof AssignmentStatement) { AssignmentStatement as = (AssignmentStatement) current; DataIdentifier target = as.getTarget(); Expression source = as.getSource(); // CASE: regular assignment statement -- source is DML expression that is NOT user-defined or external function - if (!(source instanceof FunctionCallIdentifier)){ + if(!(source instanceof FunctionCallIdentifier)) { // CASE: target is regular data identifier - if (!(target instanceof IndexedIdentifier)) { + if(!(target instanceof IndexedIdentifier)) { //process right hand side and accumulation Hop ae = processExpression(source, target, ids); - if( as.isAccumulator() ) { + if(as.isAccumulator()) { DataIdentifier accum = getAccumulatorData(liveIn, target.getName()); ae = HopRewriteUtils.createBinary(ids.get(target.getName()), ae, OpOp2.PLUS); target.setProperties(accum.getOutput()); @@ -1154,9 +1147,9 @@ else if (ptype == PRINTTYPE.STOP) { else target.setProperties(source.getOutput()); - if (source instanceof BuiltinFunctionExpression){ - BuiltinFunctionExpression BuiltinSource = (BuiltinFunctionExpression)source; - if (BuiltinSource.getOpCode() == Builtins.TIME) + if(source instanceof BuiltinFunctionExpression) { + BuiltinFunctionExpression BuiltinSource = (BuiltinFunctionExpression) source; + if(BuiltinSource.getOpCode() == Builtins.TIME) sb.setSplitDag(true); } @@ -1164,21 +1157,23 @@ else if (ptype == PRINTTYPE.STOP) { //add transient write if needed Integer statementId = liveOutToTemp.get(target.getName()); - if ((statementId != null) && (statementId.intValue() == i)) { - DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, OpOpData.TRANSIENTWRITE, null); - transientwrite.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), ae.getBlocksize()); + if((statementId != null) && (statementId.intValue() == i)) { + DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), + target.getValueType(), ae, OpOpData.TRANSIENTWRITE, null); + transientwrite.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), + ae.getBlocksize()); transientwrite.setParseInfo(target); updatedLiveOut.addVariable(target.getName(), target); output.add(transientwrite); } - } + } // CASE: target is indexed identifier (left-hand side indexed expression) else { - Hop ae = processLeftIndexedExpression(source, (IndexedIdentifier)target, ids); + Hop ae = processLeftIndexedExpression(source, (IndexedIdentifier) target, ids); - if( as.isAccumulator() ) { + if(as.isAccumulator()) { DataIdentifier accum = getAccumulatorData(liveIn, target.getName()); - Hop rix = processIndexingExpression((IndexedIdentifier)target, null, ids); + Hop rix = processIndexingExpression((IndexedIdentifier) target, null, ids); Hop rhs = processExpression(source, null, ids); Hop binary = HopRewriteUtils.createBinary(rix, rhs, OpOp2.PLUS); HopRewriteUtils.replaceChildReference(ae, ae.getInput(1), binary); @@ -1189,125 +1184,136 @@ else if (ptype == PRINTTYPE.STOP) { // obtain origDim values BEFORE they are potentially updated during setProperties call // (this is incorrect for LHS Indexing) - long origDim1 = ((IndexedIdentifier)target).getOrigDim1(); - long origDim2 = ((IndexedIdentifier)target).getOrigDim2(); + long origDim1 = ((IndexedIdentifier) target).getOrigDim1(); + long origDim2 = ((IndexedIdentifier) target).getOrigDim2(); target.setProperties(source.getOutput()); - ((IndexedIdentifier)target).setOriginalDimensions(origDim1, origDim2); + ((IndexedIdentifier) target).setOriginalDimensions(origDim1, origDim2); // preserve data type matrix of any index identifier // (required for scalar input to left indexing) - if( target.getDataType() != DataType.MATRIX ) { + if(target.getDataType() != DataType.MATRIX) { target.setDataType(DataType.MATRIX); target.setValueType(ValueType.FP64); target.setBlocksize(ConfigurationManager.getBlocksize()); } Integer statementId = liveOutToTemp.get(target.getName()); - if ((statementId != null) && (statementId.intValue() == i)) { - DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, OpOpData.TRANSIENTWRITE, null); - transientwrite.setOutputParams(origDim1, origDim2, ae.getNnz(), ae.getUpdateType(), ae.getBlocksize()); + if((statementId != null) && (statementId.intValue() == i)) { + DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), + target.getValueType(), ae, OpOpData.TRANSIENTWRITE, null); + transientwrite.setOutputParams(origDim1, origDim2, ae.getNnz(), ae.getUpdateType(), + ae.getBlocksize()); transientwrite.setParseInfo(target); updatedLiveOut.addVariable(target.getName(), target); output.add(transientwrite); } } } - else - { + else { //assignment, function call FunctionCallIdentifier fci = (FunctionCallIdentifier) source; - FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(),fci.getName()); + FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(), + fci.getName()); //error handling missing function - if (fsb == null) { - throw new LanguageException(source.printErrorLocation() + "function " - + fci.getName() + " is undefined in namespace " + fci.getNamespace()); + if(fsb == null) { + throw new LanguageException( + source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + + fci.getNamespace()); } - FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0); - String fkey = DMLProgram.constructFunctionKey(fci.getNamespace(),fci.getName()); + FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0); + String fkey = DMLProgram.constructFunctionKey(fci.getNamespace(), fci.getName()); //error handling unsupported function call in indexing expression - if( target instanceof IndexedIdentifier ) { - throw new LanguageException("Unsupported function call to '"+fkey+"' in left indexing " - + "expression. Please, assign the function output to a variable."); + if(target instanceof IndexedIdentifier) { + throw new LanguageException("Unsupported function call to '" + fkey + "' in left indexing " + + "expression. Please, assign the function output to a variable."); } //prepare function input names and inputs - List inputNames = new ArrayList<>(fci.getParamExprs().stream() - .map(e -> e.getName()).collect(Collectors.toList())); - List finputs = new ArrayList<>(fci.getParamExprs().stream() - .map(e -> processExpression(e.getExpr(), null, ids)).collect(Collectors.toList())); + List inputNames = new ArrayList<>( + fci.getParamExprs().stream().map(e -> e.getName()).collect(Collectors.toList())); + List finputs = new ArrayList<>( + fci.getParamExprs().stream().map(e -> processExpression(e.getExpr(), null, ids)) + .collect(Collectors.toList())); //append default expression for missing arguments appendDefaultArguments(fstmt, inputNames, finputs, ids); //use function signature to obtain names for unnamed args //(note: consistent parameters already checked for functions in general) - if( inputNames.stream().allMatch(n -> n==null) ) + if(inputNames.stream().allMatch(n -> n == null)) inputNames = fstmt._inputParams.stream().map(d -> d.getName()).collect(Collectors.toList()); //create function op String[] inputNames2 = inputNames.toArray(new String[0]); FunctionType ftype = fsb.getFunctionOpType(); - FunctionOp fcall = (target == null) ? - new FunctionOp(ftype, fci.getNamespace(), fci.getName(), inputNames2, finputs, new String[]{}, false) : - new FunctionOp(ftype, fci.getNamespace(), fci.getName(), inputNames2, finputs, new String[]{target.getName()}, false); + FunctionOp fcall = (target == null) ? new FunctionOp(ftype, fci.getNamespace(), fci.getName(), + inputNames2, finputs, new String[] {}, false) : new FunctionOp(ftype, fci.getNamespace(), + fci.getName(), inputNames2, finputs, new String[] {target.getName()}, false); fcall.setParseInfo(fci); output.add(fcall); } } - else if (current instanceof MultiAssignmentStatement) { + else if(current instanceof MultiAssignmentStatement) { //multi-assignment, by definition a function call MultiAssignmentStatement mas = (MultiAssignmentStatement) current; Expression source = mas.getSource(); - if ( source instanceof FunctionCallIdentifier ) { + if(source instanceof FunctionCallIdentifier) { FunctionCallIdentifier fci = (FunctionCallIdentifier) source; - FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(),fci.getName()); - if (fsb == null){ - throw new LanguageException(source.printErrorLocation() + "function " - + fci.getName() + " is undefined in namespace " + fci.getNamespace()); + FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(), + fci.getName()); + if(fsb == null) { + throw new LanguageException( + source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + + fci.getNamespace()); } - FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0); + FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0); //prepare function input names and inputs - List inputNames = new ArrayList<>(fci.getParamExprs().stream() - .map(e -> e.getName()).collect(Collectors.toList())); - List finputs = new ArrayList<>(fci.getParamExprs().stream() - .map(e -> processExpression(e.getExpr(), null, ids)).collect(Collectors.toList())); + List inputNames = new ArrayList<>( + fci.getParamExprs().stream().map(e -> e.getName()).collect(Collectors.toList())); + List finputs = new ArrayList<>( + fci.getParamExprs().stream().map(e -> processExpression(e.getExpr(), null, ids)) + .collect(Collectors.toList())); //use function signature to obtain names for unnamed args //(note: consistent parameters already checked for functions in general) - if( inputNames.stream().allMatch(n -> n==null) ) + if(inputNames.stream().allMatch(n -> n == null)) inputNames = fstmt._inputParams.stream().map(d -> d.getName()).collect(Collectors.toList()); //append default expression for missing arguments appendDefaultArguments(fstmt, inputNames, finputs, ids); //create function op - String[] foutputs = mas.getTargetList().stream() - .map(d -> d.getName()).toArray(String[]::new); + String[] foutputs = mas.getTargetList().stream().map(d -> d.getName()).toArray(String[]::new); FunctionType ftype = fsb.getFunctionOpType(); FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), inputNames.toArray(new String[0]), finputs, foutputs, false); fcall.setParseInfo(fci); output.add(fcall); } - else if ( source instanceof BuiltinFunctionExpression && ((BuiltinFunctionExpression)source).multipleReturns() ) { + else if(source instanceof BuiltinFunctionExpression && + ((BuiltinFunctionExpression) source).multipleReturns()) { // construct input hops - Hop fcall = processMultipleReturnBuiltinFunctionExpression((BuiltinFunctionExpression)source, mas.getTargetList(), ids); + Hop fcall = processMultipleReturnBuiltinFunctionExpression((BuiltinFunctionExpression) source, + mas.getTargetList(), ids); output.add(fcall); } - else if ( source instanceof ParameterizedBuiltinFunctionExpression && ((ParameterizedBuiltinFunctionExpression)source).multipleReturns() ) { + else if(source instanceof ParameterizedBuiltinFunctionExpression && + ((ParameterizedBuiltinFunctionExpression) source).multipleReturns()) { // construct input hops - Hop fcall = processMultipleReturnParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression)source, mas.getTargetList(), ids); + Hop fcall = processMultipleReturnParameterizedBuiltinFunctionExpression( + (ParameterizedBuiltinFunctionExpression) source, mas.getTargetList(), ids); output.add(fcall); } else - throw new LanguageException("Class \"" + source.getClass() + "\" is not supported in Multiple Assignment statements"); + throw new LanguageException( + "Class \"" + source.getClass() + "\" is not supported in Multiple Assignment statements"); } } @@ -1317,27 +1323,30 @@ else if ( source instanceof ParameterizedBuiltinFunctionExpression && ((Paramete private static DataIdentifier getAccumulatorData(VariableSet liveIn, String varname) { DataIdentifier accum = liveIn.getVariable(varname); - if( accum == null ) - throw new LanguageException("Invalid accumulator assignment " - + "to non-existing variable "+varname+"."); + if(accum == null) + throw new LanguageException( + "Invalid accumulator assignment " + "to non-existing variable " + varname + "."); return accum; } - private void appendDefaultArguments(FunctionStatement fstmt, List inputNames, List inputs, HashMap ids) { + private void appendDefaultArguments(FunctionStatement fstmt, List inputNames, List inputs, + HashMap ids) { //NOTE: For default expressions of unspecified function arguments, we have two choices: //either (a) compile ifelse(exist(argName),default, argName) into the function, or //simply (b) add the default to the argument list of function calls when needed. //We decided for (b) because it simplifies IPA and dynamic recompilation. - if( fstmt.getInputParams().size() == inputs.size() ) + if(fstmt.getInputParams().size() == inputs.size()) return; HashSet probeNames = new HashSet<>(inputNames); - for( DataIdentifier di : fstmt.getInputParams() ) { - if( probeNames.contains(di.getName()) ) continue; + for(DataIdentifier di : fstmt.getInputParams()) { + if(probeNames.contains(di.getName())) + continue; Expression exp = fstmt.getInputDefault(di.getName()); - if( exp == null ) { - throw new LanguageException("Missing default expression for unspecified " - + "function argument '"+di.getName()+"' in call to function '"+fstmt.getName()+"'."); + if(exp == null) { + throw new LanguageException( + "Missing default expression for unspecified " + "function argument '" + di.getName() + + "' in call to function '" + fstmt.getName() + "'."); } //compile and add default expression inputNames.add(di.getName()); @@ -1354,39 +1363,39 @@ public void constructHopsForIfControlBlock(IfStatementBlock sb) { constructHopsForConditionalPredicate(sb); // handle if statement body - for( StatementBlock current : ifBody ) { + for(StatementBlock current : ifBody) { constructHops(current); } // handle else stmt body - for( StatementBlock current : elseBody ) { + for(StatementBlock current : elseBody) { constructHops(current); } } /** * Constructs Hops for a given ForStatementBlock or ParForStatementBlock, respectively. - * + * * @param sb for statement block */ - public void constructHopsForForControlBlock(ForStatementBlock sb) { + public void constructHopsForForControlBlock(ForStatementBlock sb) { ForStatement fs = (ForStatement) sb.getStatement(0); ArrayList body = fs.getBody(); constructHopsForIterablePredicate(sb); - for( StatementBlock current : body ) + for(StatementBlock current : body) constructHops(current); } public void constructHopsForFunctionControlBlock(FunctionStatementBlock fsb) { - ArrayList body = ((FunctionStatement)fsb.getStatement(0)).getBody(); - for( StatementBlock current : body ) + ArrayList body = ((FunctionStatement) fsb.getStatement(0)).getBody(); + for(StatementBlock current : body) constructHops(current); } public void constructHopsForWhileControlBlock(WhileStatementBlock sb) { - ArrayList body = ((WhileStatement)sb.getStatement(0)).getBody(); + ArrayList body = ((WhileStatement) sb.getStatement(0)).getBody(); constructHopsForConditionalPredicate(sb); - for( StatementBlock current : body ) + for(StatementBlock current : body) constructHops(current); } @@ -1397,12 +1406,12 @@ public void constructHopsForConditionalPredicate(StatementBlock passedSB) { // set conditional predicate ConditionalPredicate cp = null; - if (passedSB instanceof WhileStatementBlock){ - WhileStatement ws = (WhileStatement) ((WhileStatementBlock)passedSB).getStatement(0); + if(passedSB instanceof WhileStatementBlock) { + WhileStatement ws = (WhileStatement) ((WhileStatementBlock) passedSB).getStatement(0); cp = ws.getConditionalPredicate(); - } - else if (passedSB instanceof IfStatementBlock) { - IfStatement ws = (IfStatement) ((IfStatementBlock)passedSB).getStatement(0); + } + else if(passedSB instanceof IfStatementBlock) { + IfStatement ws = (IfStatement) ((IfStatementBlock) passedSB).getStatement(0); cp = ws.getConditionalPredicate(); } else { @@ -1411,21 +1420,22 @@ else if (passedSB instanceof IfStatementBlock) { VariableSet varsRead = cp.variablesRead(); - for (String varName : varsRead.getVariables().keySet()) { + for(String varName : varsRead.getVariables().keySet()) { // creating transient read for live in variables DataIdentifier var = passedSB.liveIn().getVariables().get(varName); DataOp read = null; - if (var == null) { + if(var == null) { throw new ParseException("variable " + varName + " not live variable for conditional predicate"); - } else { - long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1(); - long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2(); + } + else { + long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim1() : var.getDim1(); + long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim2() : var.getDim2(); - read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), OpOpData.TRANSIENTREAD, - null, actualDim1, actualDim2, var.getNnz(), var.getBlocksize()); + read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), OpOpData.TRANSIENTREAD, null, + actualDim1, actualDim2, var.getNnz(), var.getBlocksize()); read.setParseInfo(var); } ids.put(varName, read); @@ -1438,79 +1448,83 @@ else if (passedSB instanceof IfStatementBlock) { Hop predicateHops = null; Expression predicate = cp.getPredicate(); - if (predicate instanceof RelationalExpression) { + if(predicate instanceof RelationalExpression) { predicateHops = processRelationalExpression((RelationalExpression) cp.getPredicate(), target, ids); - } else if (predicate instanceof BooleanExpression) { + } + else if(predicate instanceof BooleanExpression) { predicateHops = processBooleanExpression((BooleanExpression) cp.getPredicate(), target, ids); - } else if (predicate instanceof DataIdentifier) { + } + else if(predicate instanceof DataIdentifier) { // handle data identifier predicate predicateHops = processExpression(cp.getPredicate(), null, ids); - } else if (predicate instanceof ConstIdentifier) { + } + else if(predicate instanceof ConstIdentifier) { // handle constant identifier // a) translate 0 --> FALSE; translate 1 --> TRUE // b) disallow string values - if ((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 0) - || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 0.0)) { + if((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 0) || + (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 0.0)) { cp.setPredicate(new BooleanIdentifier(false, predicate)); - } else if ((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 1) - || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 1.0)) { + } + else if((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 1) || + (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 1.0)) { cp.setPredicate(new BooleanIdentifier(true, predicate)); - } else if (predicate instanceof IntIdentifier || predicate instanceof DoubleIdentifier) { + } + else if(predicate instanceof IntIdentifier || predicate instanceof DoubleIdentifier) { cp.setPredicate(new BooleanIdentifier(true, predicate)); - LOG.warn(predicate.printWarningLocation() + "Numerical value '" + predicate.toString() - + "' (!= 0/1) is converted to boolean TRUE by DML"); - } else if (predicate instanceof StringIdentifier) { - throw new ParseException(predicate.printErrorLocation() + "String value '" + predicate.toString() - + "' is not allowed for iterable predicate"); + LOG.warn(predicate.printWarningLocation() + "Numerical value '" + predicate.toString() + + "' (!= 0/1) is converted to boolean TRUE by DML"); + } + else if(predicate instanceof StringIdentifier) { + throw new ParseException(predicate.printErrorLocation() + "String value '" + predicate.toString() + + "' is not allowed for iterable predicate"); } predicateHops = processExpression(cp.getPredicate(), null, ids); } //create transient write to internal variable name on top of expression //in order to ensure proper instruction generation - predicateHops = HopRewriteUtils.createDataOp( - ProgramBlock.PRED_VAR, predicateHops, OpOpData.TRANSIENTWRITE); + predicateHops = HopRewriteUtils.createDataOp(ProgramBlock.PRED_VAR, predicateHops, OpOpData.TRANSIENTWRITE); - if (passedSB instanceof WhileStatementBlock) - ((WhileStatementBlock)passedSB).setPredicateHops(predicateHops); - else if (passedSB instanceof IfStatementBlock) - ((IfStatementBlock)passedSB).setPredicateHops(predicateHops); + if(passedSB instanceof WhileStatementBlock) + ((WhileStatementBlock) passedSB).setPredicateHops(predicateHops); + else if(passedSB instanceof IfStatementBlock) + ((IfStatementBlock) passedSB).setPredicateHops(predicateHops); } /** - * Constructs all predicate Hops (for FROM, TO, INCREMENT) of an iterable predicate - * and assigns these Hops to the passed statement block. - * + * Constructs all predicate Hops (for FROM, TO, INCREMENT) of an iterable predicate and assigns these Hops to the + * passed statement block. + * * Method used for both ForStatementBlock and ParForStatementBlock. - * + * * @param fsb for statement block */ - public void constructHopsForIterablePredicate(ForStatementBlock fsb) - { + public void constructHopsForIterablePredicate(ForStatementBlock fsb) { HashMap ids = new HashMap<>(); // set iterable predicate ForStatement fs = (ForStatement) fsb.getStatement(0); IterablePredicate ip = fs.getIterablePredicate(); - for(int i=0; i < 3; i++) { - Expression expr = (i == 0) ? ip.getFromExpr() : (i == 1) ? ip.getToExpr() : - ( ip.getIncrementExpr() != null ) ? ip.getIncrementExpr() : null; + for(int i = 0; i < 3; i++) { + Expression expr = (i == 0) ? ip.getFromExpr() : (i == 1) ? ip.getToExpr() : (ip.getIncrementExpr() != + null) ? ip.getIncrementExpr() : null; VariableSet varsRead = (expr != null) ? expr.variablesRead() : null; if(varsRead != null) { - for (String varName : varsRead.getVariables().keySet()) { + for(String varName : varsRead.getVariables().keySet()) { DataIdentifier var = fsb.liveIn().getVariable(varName); DataOp read = null; - if (var == null) { + if(var == null) { throw new ParseException("variable '" + varName + "' is not available for iterable predicate"); } else { - long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1(); - long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2(); + long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim1() : var.getDim1(); + long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim2() : var.getDim2(); read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), OpOpData.TRANSIENTREAD, - null, actualDim1, actualDim2, var.getNnz(), var.getBlocksize()); + null, actualDim1, actualDim2, var.getNnz(), var.getBlocksize()); read.setParseInfo(var); } ids.put(varName, read); @@ -1520,111 +1534,111 @@ public void constructHopsForIterablePredicate(ForStatementBlock fsb) //create transient write to internal variable name on top of expression //in order to ensure proper instruction generation Hop predicateHops = processTempIntExpression(expr, ids); - if( predicateHops != null ) - predicateHops = HopRewriteUtils.createDataOp( - ProgramBlock.PRED_VAR, predicateHops, OpOpData.TRANSIENTWRITE); + if(predicateHops != null) + predicateHops = HopRewriteUtils.createDataOp(ProgramBlock.PRED_VAR, predicateHops, + OpOpData.TRANSIENTWRITE); //construct hops for from, to, and increment expressions - if( i == 0 ) - fsb.setFromHops( predicateHops ); - else if( i == 1 ) - fsb.setToHops( predicateHops ); - else if( ip.getIncrementExpr() != null ) - fsb.setIncrementHops( predicateHops ); + if(i == 0) + fsb.setFromHops(predicateHops); + else if(i == 1) + fsb.setToHops(predicateHops); + else if(ip.getIncrementExpr() != null) + fsb.setIncrementHops(predicateHops); } } /** - * Construct Hops from parse tree : Process Expression in an assignment - * statement - * + * Construct Hops from parse tree : Process Expression in an assignment statement + * * @param source source expression * @param target data identifier - * @param hops map of high-level operators + * @param hops map of high-level operators * @return high-level operator */ private Hop processExpression(Expression source, DataIdentifier target, HashMap hops) { try { - if( source instanceof BinaryExpression ) + if(source instanceof BinaryExpression) return processBinaryExpression((BinaryExpression) source, target, hops); - else if( source instanceof RelationalExpression ) + else if(source instanceof RelationalExpression) return processRelationalExpression((RelationalExpression) source, target, hops); - else if( source instanceof BooleanExpression ) + else if(source instanceof BooleanExpression) return processBooleanExpression((BooleanExpression) source, target, hops); - else if( source instanceof BuiltinFunctionExpression ) + else if(source instanceof BuiltinFunctionExpression) return processBuiltinFunctionExpression((BuiltinFunctionExpression) source, target, hops); - else if( source instanceof ParameterizedBuiltinFunctionExpression ) - return processParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression)source, target, hops); - else if( source instanceof DataExpression ) { - Hop ae = processDataExpression((DataExpression)source, target, hops); - if (ae instanceof DataOp && ((DataOp) ae).getOp() != OpOpData.SQLREAD && - ((DataOp) ae).getOp() != OpOpData.FEDERATED) { - Expression expr = ((DataExpression)source).getVarParam(DataExpression.FORMAT_TYPE); - if( expr instanceof StringIdentifier ) - ((DataOp)ae).setFileFormat(Expression.convertFormatType(expr.toString())); + else if(source instanceof ParameterizedBuiltinFunctionExpression) + return processParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression) source, + target, hops); + else if(source instanceof DataExpression) { + Hop ae = processDataExpression((DataExpression) source, target, hops); + if(ae instanceof DataOp && ((DataOp) ae).getOp() != OpOpData.SQLREAD && + ((DataOp) ae).getOp() != OpOpData.FEDERATED) { + Expression expr = ((DataExpression) source).getVarParam(DataExpression.FORMAT_TYPE); + if(expr instanceof StringIdentifier) + ((DataOp) ae).setFileFormat(Expression.convertFormatType(expr.toString())); else - ((DataOp)ae).setFileFormat(FileFormat.UNKNOWN); + ((DataOp) ae).setFileFormat(FileFormat.UNKNOWN); } return ae; } - else if (source instanceof IndexedIdentifier) - return processIndexingExpression((IndexedIdentifier) source,target,hops); - else if (source instanceof IntIdentifier) { + else if(source instanceof IndexedIdentifier) + return processIndexingExpression((IndexedIdentifier) source, target, hops); + else if(source instanceof IntIdentifier) { IntIdentifier sourceInt = (IntIdentifier) source; LiteralOp litop = new LiteralOp(sourceInt.getValue()); litop.setParseInfo(sourceInt); setIdentifierParams(litop, sourceInt); return litop; - } - else if (source instanceof DoubleIdentifier) { + } + else if(source instanceof DoubleIdentifier) { DoubleIdentifier sourceDouble = (DoubleIdentifier) source; LiteralOp litop = new LiteralOp(sourceDouble.getValue()); litop.setParseInfo(sourceDouble); setIdentifierParams(litop, sourceDouble); return litop; } - else if (source instanceof BooleanIdentifier) { + else if(source instanceof BooleanIdentifier) { BooleanIdentifier sourceBoolean = (BooleanIdentifier) source; LiteralOp litop = new LiteralOp(sourceBoolean.getValue()); litop.setParseInfo(sourceBoolean); setIdentifierParams(litop, sourceBoolean); return litop; - } - else if (source instanceof StringIdentifier) { + } + else if(source instanceof StringIdentifier) { StringIdentifier sourceString = (StringIdentifier) source; LiteralOp litop = new LiteralOp(sourceString.getValue()); litop.setParseInfo(sourceString); setIdentifierParams(litop, sourceString); return litop; - } - else if (source instanceof DataIdentifier) + } + else if(source instanceof DataIdentifier) return hops.get(((DataIdentifier) source).getName()); - else if (source instanceof ExpressionList){ + else if(source instanceof ExpressionList) { ExpressionList sourceList = (ExpressionList) source; List expressions = sourceList.getValue(); Hop[] listHops = new Hop[expressions.size()]; int idx = 0; - for( Expression ex : expressions){ + for(Expression ex : expressions) { listHops[idx++] = processExpression(ex, null, hops); } - Hop currBuiltinOp = HopRewriteUtils.createNary(OpOpN.LIST,listHops ); + Hop currBuiltinOp = HopRewriteUtils.createNary(OpOpN.LIST, listHops); return currBuiltinOp; } - else{ + else { throw new ParseException("Unhandled instance of source type: " + source); } - } - catch(ParseException e ){ + } + catch(ParseException e) { throw e; } - catch ( Exception e ) { + catch(Exception e) { throw new ParseException("A Parsing exception occurred", e); } } private static DataIdentifier createTarget(Expression source) { Identifier id = source.getOutput(); - if (id instanceof DataIdentifier && !(id instanceof DataExpression)) + if(id instanceof DataIdentifier && !(id instanceof DataExpression)) return (DataIdentifier) id; DataIdentifier target = new DataIdentifier(Expression.getTempName()); target.setProperties(id); @@ -1636,20 +1650,20 @@ private static DataIdentifier createTarget() { } /** - * Constructs the Hops for arbitrary expressions that eventually evaluate to an INT scalar. - * + * Constructs the Hops for arbitrary expressions that eventually evaluate to an INT scalar. + * * @param source source expression - * @param hops map of high-level operators + * @param hops map of high-level operators * @return high-level operatos */ - private Hop processTempIntExpression( Expression source, HashMap hops ) { - if( source == null ) + private Hop processTempIntExpression(Expression source, HashMap hops) { + if(source == null) return null; DataIdentifier tmpOut = createTarget(); tmpOut.setDataType(DataType.SCALAR); tmpOut.setValueType(ValueType.INT64); source.setOutput(tmpOut); - return processExpression(source, tmpOut, hops ); + return processExpression(source, tmpOut, hops); } private Hop processLeftIndexedExpression(Expression source, IndexedIdentifier target, HashMap hops) { @@ -1661,16 +1675,17 @@ private Hop processLeftIndexedExpression(Expression source, IndexedIdentifier ta // process the target to get targetHops Hop targetOp = hops.get(target.getName()); - if (targetOp == null){ - throw new ParseException(target.printErrorLocation() + " must define matrix " + target.getName() + " before indexing operations are allowed "); + if(targetOp == null) { + throw new ParseException(target.printErrorLocation() + " must define matrix " + target.getName() + + " before indexing operations are allowed "); } - if( sourceOp.getDataType().isMatrix() && source.getOutput().getDataType().isScalar() ) + if(sourceOp.getDataType().isMatrix() && source.getOutput().getDataType().isScalar()) sourceOp.setDataType(DataType.SCALAR); - Hop leftIndexOp = new LeftIndexingOp(target.getName(), target.getDataType(), - ValueType.FP64, targetOp, sourceOp, ixRange[0], ixRange[1], ixRange[2], ixRange[3], - target.getRowLowerEqualsUpper(), target.getColLowerEqualsUpper()); + Hop leftIndexOp = new LeftIndexingOp(target.getName(), target.getDataType(), ValueType.FP64, targetOp, sourceOp, + ixRange[0], ixRange[1], ixRange[2], ixRange[3], target.getRowLowerEqualsUpper(), + target.getColLowerEqualsUpper()); setIdentifierParams(leftIndexOp, target); leftIndexOp.setParseInfo(target); @@ -1684,18 +1699,18 @@ private Hop processIndexingExpression(IndexedIdentifier source, DataIdentifier t // process Hops for indexes (for source) Hop[] ixRange = getIndexingBounds(source, hops, false); - if (target == null) { + if(target == null) { target = createTarget(source); } - + //unknown nnz after range indexing (applies to indexing op but also //data dependent operations) - target.setNnz(-1); + target.setNnz(-1); DataType dt = target.getDataType().isScalar() ? DataType.MATRIX : target.getDataType(); - Hop indexOp = new IndexingOp(target.getName(), dt, target.getValueType(), - hops.get(source.getName()), ixRange[0], ixRange[1], ixRange[2], ixRange[3], - source.getRowLowerEqualsUpper(), source.getColLowerEqualsUpper()); + Hop indexOp = new IndexingOp(target.getName(), dt, target.getValueType(), hops.get(source.getName()), + ixRange[0], ixRange[1], ixRange[2], ixRange[3], source.getRowLowerEqualsUpper(), + source.getColLowerEqualsUpper()); indexOp.setParseInfo(target); setIdentifierParams(indexOp, target); @@ -1704,27 +1719,27 @@ private Hop processIndexingExpression(IndexedIdentifier source, DataIdentifier t } private Hop[] getIndexingBounds(IndexedIdentifier ix, HashMap hops, boolean lix) { - Hop rowLowerHops = (ix.getRowLowerBound() != null) ? - processExpression(ix.getRowLowerBound(),null, hops) : new LiteralOp(1); - Hop colLowerHops = (ix.getColLowerBound() != null) ? - processExpression(ix.getColLowerBound(),null, hops) : new LiteralOp(1); + Hop rowLowerHops = (ix.getRowLowerBound() != null) ? processExpression(ix.getRowLowerBound(), null, + hops) : new LiteralOp(1); + Hop colLowerHops = (ix.getColLowerBound() != null) ? processExpression(ix.getColLowerBound(), null, + hops) : new LiteralOp(1); Hop rowUpperHops = null, colUpperHops = null; - if (ix.getRowUpperBound() != null) - rowUpperHops = processExpression(ix.getRowUpperBound(),null,hops); + if(ix.getRowUpperBound() != null) + rowUpperHops = processExpression(ix.getRowUpperBound(), null, hops); else { - rowUpperHops = ((lix ? ix.getDim1() : ix.getOrigDim1()) != -1) ? - new LiteralOp(ix.getOrigDim1()) : - new UnaryOp(ix.getName(), DataType.SCALAR, ValueType.INT64, OpOp1.NROW, hops.get(ix.getName())); + rowUpperHops = ((lix ? ix.getDim1() : ix.getOrigDim1()) != -1) ? new LiteralOp( + ix.getOrigDim1()) : new UnaryOp(ix.getName(), DataType.SCALAR, ValueType.INT64, OpOp1.NROW, + hops.get(ix.getName())); rowUpperHops.setParseInfo(ix); } - if (ix.getColUpperBound() != null) - colUpperHops = processExpression(ix.getColUpperBound(),null,hops); + if(ix.getColUpperBound() != null) + colUpperHops = processExpression(ix.getColUpperBound(), null, hops); else { - colUpperHops = ((lix ? ix.getDim2() : ix.getOrigDim2()) != -1) ? - new LiteralOp(ix.getOrigDim2()) : - new UnaryOp(ix.getName(), DataType.SCALAR, ValueType.INT64, OpOp1.NCOL, hops.get(ix.getName())); + colUpperHops = ((lix ? ix.getDim2() : ix.getOrigDim2()) != -1) ? new LiteralOp( + ix.getOrigDim2()) : new UnaryOp(ix.getName(), DataType.SCALAR, ValueType.INT64, OpOp1.NCOL, + hops.get(ix.getName())); colUpperHops.setParseInfo(ix); } @@ -1732,33 +1747,31 @@ private Hop[] getIndexingBounds(IndexedIdentifier ix, HashMap hops, } /** - * Construct Hops from parse tree : Process Binary Expression in an - * assignment statement - * + * Construct Hops from parse tree : Process Binary Expression in an assignment statement + * * @param source binary expression * @param target data identifier - * @param hops map of high-level operators + * @param hops map of high-level operators * @return high-level operator */ - private Hop processBinaryExpression(BinaryExpression source, DataIdentifier target, HashMap hops) - { - Hop left = processExpression(source.getLeft(), null, hops); + private Hop processBinaryExpression(BinaryExpression source, DataIdentifier target, HashMap hops) { + Hop left = processExpression(source.getLeft(), null, hops); Hop right = processExpression(source.getRight(), null, hops); - if (left == null || right == null) { - throw new ParseException("Missing input in binary expressions (" + source.toString()+"): " - + ((left==null)?source.getLeft():source.getRight())+", line="+source.getBeginLine()); + if(left == null || right == null) { + throw new ParseException("Missing input in binary expressions (" + source.toString() + "): " + + ((left == null) ? source.getLeft() : source.getRight()) + ", line=" + source.getBeginLine()); } //prepare target identifier and ensure that output type is of inferred type //(type should not be determined by target (e.g., string for print) - if (target == null) { + if(target == null) { target = createTarget(source); } target.setValueType(source.getOutput().getValueType()); Hop currBop = null; - switch( source.getOpCode() ) { + switch(source.getOpCode()) { case PLUS: case MINUS: case MULT: @@ -1766,14 +1779,15 @@ private Hop processBinaryExpression(BinaryExpression source, DataIdentifier targ case MODULUS: case POW: case INTDIV: - currBop = new BinaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), left, right); + currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp2.valueOf(source.getOpCode().name()), left, right); break; case MATMULT: - currBop = new AggBinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MULT, org.apache.sysds.common.Types.AggOp.SUM, left, right); + currBop = new AggBinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MULT, + org.apache.sysds.common.Types.AggOp.SUM, left, right); break; default: - throw new ParseException("Unsupported parsing of binary expression: "+source.getOpCode()); + throw new ParseException("Unsupported parsing of binary expression: " + source.getOpCode()); } setIdentifierParams(currBop, source.getOutput()); @@ -1781,14 +1795,15 @@ private Hop processBinaryExpression(BinaryExpression source, DataIdentifier targ return currBop; } - private Hop processRelationalExpression(RelationalExpression source, DataIdentifier target, HashMap hops) { + private Hop processRelationalExpression(RelationalExpression source, DataIdentifier target, + HashMap hops) { Hop left = processExpression(source.getLeft(), null, hops); Hop right = processExpression(source.getRight(), null, hops); Hop currBop = null; - if (target == null) { + if(target == null) { target = createTarget(source); if(left.getDataType() == DataType.MATRIX || right.getDataType() == DataType.MATRIX) { // Added to support matrix relational comparison @@ -1809,17 +1824,22 @@ else if(left.getDataType() == DataType.FRAME || right.getDataType() == DataType. OpOp2 op = null; - if (source.getOpCode() == Expression.RelationalOp.LESS) { + if(source.getOpCode() == Expression.RelationalOp.LESS) { op = OpOp2.LESS; - } else if (source.getOpCode() == Expression.RelationalOp.LESSEQUAL) { + } + else if(source.getOpCode() == Expression.RelationalOp.LESSEQUAL) { op = OpOp2.LESSEQUAL; - } else if (source.getOpCode() == Expression.RelationalOp.GREATER) { + } + else if(source.getOpCode() == Expression.RelationalOp.GREATER) { op = OpOp2.GREATER; - } else if (source.getOpCode() == Expression.RelationalOp.GREATEREQUAL) { + } + else if(source.getOpCode() == Expression.RelationalOp.GREATEREQUAL) { op = OpOp2.GREATEREQUAL; - } else if (source.getOpCode() == Expression.RelationalOp.EQUAL) { + } + else if(source.getOpCode() == Expression.RelationalOp.EQUAL) { op = OpOp2.EQUAL; - } else if (source.getOpCode() == Expression.RelationalOp.NOTEQUAL) { + } + else if(source.getOpCode() == Expression.RelationalOp.NOTEQUAL) { op = OpOp2.NOTEQUAL; } currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), op, left, right); @@ -1827,45 +1847,47 @@ else if(left.getDataType() == DataType.FRAME || right.getDataType() == DataType. return currBop; } - private Hop processBooleanExpression(BooleanExpression source, DataIdentifier target, HashMap hops) - { + private Hop processBooleanExpression(BooleanExpression source, DataIdentifier target, HashMap hops) { // Boolean Not has a single parameter boolean constLeft = (source.getLeft().getOutput() instanceof ConstIdentifier); boolean constRight = false; - if (source.getRight() != null) { + if(source.getRight() != null) { constRight = (source.getRight().getOutput() instanceof ConstIdentifier); } - Hop left = !constLeft ? processExpression(source.getLeft(), null, hops) : - new LiteralOp(Boolean.valueOf(source.getLeft().getText().toLowerCase())); + Hop left = !constLeft ? processExpression(source.getLeft(), null, hops) : new LiteralOp( + Boolean.valueOf(source.getLeft().getText().toLowerCase())); Hop right = null; - if (source.getRight() != null) { - right = !constRight ? processExpression(source.getRight(), null, hops) : - new LiteralOp(Boolean.valueOf(source.getRight().getText().toLowerCase())); + if(source.getRight() != null) { + right = !constRight ? processExpression(source.getRight(), null, hops) : new LiteralOp( + Boolean.valueOf(source.getRight().getText().toLowerCase())); } //prepare target identifier and ensure that output type is boolean //(type should not be determined by target (e.g., string for print) - if (target == null) + if(target == null) target = createTarget(source); - if( target.getDataType().isScalar() ) + if(target.getDataType().isScalar()) target.setValueType(ValueType.BOOLEAN); - if (source.getRight() == null) { + if(source.getRight() == null) { Hop currUop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp1.NOT, left); currUop.setParseInfo(source); return currUop; - } + } else { Hop currBop = null; OpOp2 op = null; - if (source.getOpCode() == Expression.BooleanOp.LOGICALAND) { + if(source.getOpCode() == Expression.BooleanOp.LOGICALAND) { op = OpOp2.AND; - } else if (source.getOpCode() == Expression.BooleanOp.LOGICALOR) { + } + else if(source.getOpCode() == Expression.BooleanOp.LOGICALOR) { op = OpOp2.OR; - } else { - throw new RuntimeException(source.printErrorLocation() + "Unknown boolean operation " + source.getOpCode()); + } + else { + throw new RuntimeException( + source.printErrorLocation() + "Unknown boolean operation " + source.getOpCode()); } currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), op, left, right); currBop.setParseInfo(source); @@ -1874,49 +1896,51 @@ private Hop processBooleanExpression(BooleanExpression source, DataIdentifier ta } } - private static Hop constructDfHop(String name, DataType dt, ValueType vt, Builtins op, LinkedHashMap paramHops) { + private static Hop constructDfHop(String name, DataType dt, ValueType vt, Builtins op, + LinkedHashMap paramHops) { // Add a hop to paramHops to store distribution information. // Distribution parameter hops would have been already present in paramHops. Hop distLop = null; switch(op) { - case QNORM: - case PNORM: - distLop = new LiteralOp("normal"); - break; - case QT: - case PT: - distLop = new LiteralOp("t"); - break; - case QF: - case PF: - distLop = new LiteralOp("f"); - break; - case QCHISQ: - case PCHISQ: - distLop = new LiteralOp("chisq"); - break; - case QEXP: - case PEXP: - distLop = new LiteralOp("exp"); - break; - - case CDF: - case INVCDF: - break; - - default: - throw new HopsException("Invalid operation: " + op); - } - if (distLop != null) + case QNORM: + case PNORM: + distLop = new LiteralOp("normal"); + break; + case QT: + case PT: + distLop = new LiteralOp("t"); + break; + case QF: + case PF: + distLop = new LiteralOp("f"); + break; + case QCHISQ: + case PCHISQ: + distLop = new LiteralOp("chisq"); + break; + case QEXP: + case PEXP: + distLop = new LiteralOp("exp"); + break; + + case CDF: + case INVCDF: + break; + + default: + throw new HopsException("Invalid operation: " + op); + } + if(distLop != null) paramHops.put("dist", distLop); - return new ParameterizedBuiltinOp(name, dt, vt, ParameterizedBuiltinFunctionExpression.pbHopMap.get(op), paramHops); + return new ParameterizedBuiltinOp(name, dt, vt, ParameterizedBuiltinFunctionExpression.pbHopMap.get(op), + paramHops); } - private Hop processMultipleReturnParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionExpression source, ArrayList targetList, - HashMap hops) - { + private Hop processMultipleReturnParameterizedBuiltinFunctionExpression( + ParameterizedBuiltinFunctionExpression source, ArrayList targetList, + HashMap hops) { FunctionType ftype = FunctionType.MULTIRETURN_BUILTIN; String nameSpace = DMLProgram.INTERNAL_NAMESPACE; @@ -1926,27 +1950,34 @@ private Hop processMultipleReturnParameterizedBuiltinFunctionExpression(Paramete // Construct Hop for current builtin function expression based on its type Hop currBuiltinOp = null; - switch (source.getOpCode()) { + switch(source.getOpCode()) { case TRANSFORMENCODE: ArrayList inputs = new ArrayList<>(); - inputs.add( processExpression(source.getVarParam("target"), null, hops) ); - inputs.add( processExpression(source.getVarParam("spec"), null, hops) ); - String[] outputNames = new String[targetList.size()]; + inputs.add(processExpression(source.getVarParam("target"), null, hops)); + inputs.add(processExpression(source.getVarParam("spec"), null, hops)); + String[] outputNames = new String[targetList.size()]; outputNames[0] = targetList.get(0).getName(); outputNames[1] = targetList.get(1).getName(); - outputs.add(new DataOp(outputNames[0], DataType.MATRIX, ValueType.FP64, inputs.get(0), OpOpData.FUNCTIONOUTPUT, inputs.get(0).getFilename())); - outputs.add(new DataOp(outputNames[1], DataType.FRAME, ValueType.STRING, inputs.get(0), OpOpData.FUNCTIONOUTPUT, inputs.get(0).getFilename())); - - currBuiltinOp = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), null, inputs, outputNames, outputs); + outputs.add( + new DataOp(outputNames[0], DataType.MATRIX, ValueType.FP64, inputs.get(0), OpOpData.FUNCTIONOUTPUT, + inputs.get(0).getFilename())); + outputs.add( + new DataOp(outputNames[1], DataType.FRAME, ValueType.STRING, inputs.get(0), OpOpData.FUNCTIONOUTPUT, + inputs.get(0).getFilename())); + + currBuiltinOp = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), null, inputs, + outputNames, outputs); break; default: - throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnParameterizedBuiltinFunctionExpression(): " + source.getOpCode()); + throw new ParseException( + "Invaid Opcode in DMLTranslator:processMultipleReturnParameterizedBuiltinFunctionExpression(): " + + source.getOpCode()); } // set properties for created hops based on outputs of source expression - for ( int i=0; i < source.getOutputs().length; i++ ) { - setIdentifierParams( outputs.get(i), source.getOutputs()[i]); + for(int i = 0; i < source.getOutputs().length; i++) { + setIdentifierParams(outputs.get(i), source.getOutputs()[i]); outputs.get(i).setParseInfo(source); } currBuiltinOp.setParseInfo(source); @@ -1955,16 +1986,15 @@ private Hop processMultipleReturnParameterizedBuiltinFunctionExpression(Paramete } /** - * Construct Hops from parse tree : Process ParameterizedBuiltinFunction Expression in an - * assignment statement - * + * Construct Hops from parse tree : Process ParameterizedBuiltinFunction Expression in an assignment statement + * * @param source parameterized built-in function * @param target data identifier - * @param hops map of high-level operators + * @param hops map of high-level operators * @return high-level operator */ - private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionExpression source, DataIdentifier target, - HashMap hops) { + private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionExpression source, + DataIdentifier target, HashMap hops) { // this expression has multiple "named" parameters LinkedHashMap paramHops = new LinkedHashMap<>(); @@ -1972,14 +2002,14 @@ private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFu // -- construct hops for all input parameters // -- store them in hashmap so that their "name"s are maintained Hop pHop = null; - for ( String paramName : source.getVarParams().keySet() ) { + for(String paramName : source.getVarParams().keySet()) { pHop = processExpression(source.getVarParam(paramName), null, hops); paramHops.put(paramName, pHop); } Hop currBuiltinOp = null; - if (target == null) { + if(target == null) { target = createTarget(source); } @@ -1997,8 +2027,8 @@ private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFu case PF: case PCHISQ: case PEXP: - currBuiltinOp = constructDfHop(target.getName(), target.getDataType(), - target.getValueType(), source.getOpCode(), paramHops); + currBuiltinOp = constructDfHop(target.getName(), target.getDataType(), target.getValueType(), + source.getOpCode(), paramHops); break; case CONTAINS: case GROUPEDAGG: @@ -2023,16 +2053,16 @@ private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFu inputs.add(paramHops.get("by")); inputs.add(paramHops.get("decreasing")); inputs.add(paramHops.get("index.return")); - currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), ReOrgOp.SORT, inputs); + currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), ReOrgOp.SORT, + inputs); break; case TOSTRING: //check for input data type and only compile toString Hop for matrices/frames, //for scalars, we compile (s + "") to ensure consistent string output value types - currBuiltinOp = !paramHops.get("target").getDataType().isScalar() ? - new ParameterizedBuiltinOp(target.getName(), target.getDataType(), - target.getValueType(), ParamBuiltinOp.TOSTRING, paramHops) : - HopRewriteUtils.createBinary(paramHops.get("target"), new LiteralOp(""), OpOp2.PLUS); + currBuiltinOp = !paramHops.get("target").getDataType().isScalar() ? new ParameterizedBuiltinOp( + target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.TOSTRING, + paramHops) : HopRewriteUtils.createBinary(paramHops.get("target"), new LiteralOp(""), OpOp2.PLUS); break; case LISTNV: @@ -2046,33 +2076,35 @@ private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFu DataType dataType = DataType.SCALAR; // Default output data type LiteralOp dirOp = (LiteralOp) paramHops.get("dir"); - if (dirOp != null) { + if(dirOp != null) { String dirString = dirOp.getStringValue().toUpperCase(); - if (dirString.equals(Direction.RowCol.toString())) { + if(dirString.equals(Direction.RowCol.toString())) { dir = Direction.RowCol; dataType = DataType.SCALAR; - } else if (dirString.equals(Direction.Row.toString())) { + } + else if(dirString.equals(Direction.Row.toString())) { dir = Direction.Row; dataType = DataType.MATRIX; - } else if (dirString.equals(Direction.Col.toString())) { + } + else if(dirString.equals(Direction.Col.toString())) { dir = Direction.Col; dataType = DataType.MATRIX; } } currBuiltinOp = new AggUnaryOp(target.getName(), dataType, target.getValueType(), - AggOp.valueOf(source.getOpCode().name()), dir, paramHops.get("data")); + AggOp.valueOf(source.getOpCode().name()), dir, paramHops.get("data")); break; } case COUNT_DISTINCT_APPROX_ROW: currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), - AggOp.COUNT_DISTINCT_APPROX, Direction.Row, paramHops.get("data")); + AggOp.COUNT_DISTINCT_APPROX, Direction.Row, paramHops.get("data")); break; case COUNT_DISTINCT_APPROX_COL: currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), - AggOp.COUNT_DISTINCT_APPROX, Direction.Col, paramHops.get("data")); + AggOp.COUNT_DISTINCT_APPROX, Direction.Col, paramHops.get("data")); break; case UNIQUE: @@ -2080,23 +2112,25 @@ private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFu DataType dataType = DataType.MATRIX; LiteralOp dirOp = (LiteralOp) paramHops.get("dir"); - if (dirOp != null) { + if(dirOp != null) { String dirString = dirOp.getStringValue().toUpperCase(); - if (dirString.equals(Direction.RowCol.toString())) { + if(dirString.equals(Direction.RowCol.toString())) { dir = Direction.RowCol; - } else if (dirString.equals(Direction.Row.toString())) { + } + else if(dirString.equals(Direction.Row.toString())) { dir = Direction.Row; - } else if (dirString.equals(Direction.Col.toString())) { + } + else if(dirString.equals(Direction.Col.toString())) { dir = Direction.Col; } } currBuiltinOp = new AggUnaryOp(target.getName(), dataType, target.getValueType(), - AggOp.valueOf(source.getOpCode().name()), dir, paramHops.get("data")); + AggOp.valueOf(source.getOpCode().name()), dir, paramHops.get("data")); break; default: - throw new ParseException(source.printErrorLocation() + + throw new ParseException(source.printErrorLocation() + "processParameterizedBuiltinFunctionExpression() -- Unknown operation: " + source.getOpCode()); } @@ -2106,95 +2140,96 @@ private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFu } /** - * Construct Hops from parse tree : Process ParameterizedExpression in a - * read/write/rand statement - * + * Construct Hops from parse tree : Process ParameterizedExpression in a read/write/rand statement + * * @param source data expression * @param target data identifier - * @param hops map of high-level operators + * @param hops map of high-level operators * @return high-level operator */ - private Hop processDataExpression(DataExpression source, DataIdentifier target, - HashMap hops) { + private Hop processDataExpression(DataExpression source, DataIdentifier target, HashMap hops) { // this expression has multiple "named" parameters HashMap paramHops = new HashMap<>(); // -- construct hops for all input parameters // -- store them in hashmap so that their "name"s are maintained - Hop pHop = null; - for ( String paramName : source.getVarParams().keySet() ) { + Hop pHop = null; + for(String paramName : source.getVarParams().keySet()) { pHop = processExpression(source.getVarParam(paramName), null, hops); paramHops.put(paramName, pHop); } Hop currBuiltinOp = null; - if (target == null) { + if(target == null) { target = createTarget(source); } // construct hop based on opcode switch(source.getOpCode()) { - case READ: - currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), OpOpData.PERSISTENTREAD, paramHops); - ((DataOp)currBuiltinOp).setFileName(((StringIdentifier)source.getVarParam(DataExpression.IO_FILENAME)).getValue()); - break; - - case WRITE: - currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), - OpOpData.PERSISTENTWRITE, hops.get(target.getName()), paramHops); - break; - - case RAND: - // We limit RAND_MIN, RAND_MAX, RAND_SPARSITY, RAND_SEED, and RAND_PDF to be constants - OpOpDG method = (paramHops.get(DataExpression.RAND_MIN).getValueType()==ValueType.STRING && + case READ: + currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), + OpOpData.PERSISTENTREAD, paramHops); + ((DataOp) currBuiltinOp).setFileName( + ((StringIdentifier) source.getVarParam(DataExpression.IO_FILENAME)).getValue()); + break; + + case WRITE: + currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), + OpOpData.PERSISTENTWRITE, hops.get(target.getName()), paramHops); + break; + + case RAND: + // We limit RAND_MIN, RAND_MAX, RAND_SPARSITY, RAND_SEED, and RAND_PDF to be constants + OpOpDG method = (paramHops.get(DataExpression.RAND_MIN).getValueType() == ValueType.STRING && target.getDataType() == DataType.MATRIX) ? OpOpDG.SINIT : OpOpDG.RAND; - currBuiltinOp = new DataGenOp(method, target, paramHops); - break; - - case FRAME: - // We limit RAND_MIN, RAND_MAX, RAND_SPARSITY, RAND_SEED, and RAND_PDF to be constants - method = OpOpDG.FRAMEINIT; - currBuiltinOp = new DataGenOp(method, target, paramHops); - break; - - case TENSOR: - case MATRIX: - ArrayList tmpMatrix = new ArrayList<>(); - tmpMatrix.add( 0, paramHops.get(DataExpression.RAND_DATA) ); - tmpMatrix.add( 1, paramHops.get(DataExpression.RAND_ROWS) ); - tmpMatrix.add( 2, paramHops.get(DataExpression.RAND_COLS) ); - tmpMatrix.add( 3, !paramHops.containsKey(DataExpression.RAND_DIMS) ? - new LiteralOp("-1") : paramHops.get(DataExpression.RAND_DIMS)); - tmpMatrix.add( 4, paramHops.get(DataExpression.RAND_BY_ROW) ); - currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), - target.getValueType(), ReOrgOp.RESHAPE, tmpMatrix); - break; - - case SQL: - currBuiltinOp = new DataOp(target.getName(), target.getDataType(), - target.getValueType(), OpOpData.SQLREAD, paramHops); - break; - - case FEDERATED: - currBuiltinOp = new DataOp(target.getName(), target.getDataType(), - target.getValueType(), OpOpData.FEDERATED, paramHops); - break; - - default: - throw new ParseException(source.printErrorLocation() + - "processDataExpression():: Unknown operation: " + source.getOpCode()); + currBuiltinOp = new DataGenOp(method, target, paramHops); + break; + + case FRAME: + // We limit RAND_MIN, RAND_MAX, RAND_SPARSITY, RAND_SEED, and RAND_PDF to be constants + method = OpOpDG.FRAMEINIT; + currBuiltinOp = new DataGenOp(method, target, paramHops); + break; + + case TENSOR: + case MATRIX: + ArrayList tmpMatrix = new ArrayList<>(); + tmpMatrix.add(0, paramHops.get(DataExpression.RAND_DATA)); + tmpMatrix.add(1, paramHops.get(DataExpression.RAND_ROWS)); + tmpMatrix.add(2, paramHops.get(DataExpression.RAND_COLS)); + tmpMatrix.add(3, !paramHops.containsKey(DataExpression.RAND_DIMS) ? new LiteralOp("-1") : paramHops.get( + DataExpression.RAND_DIMS)); + tmpMatrix.add(4, paramHops.get(DataExpression.RAND_BY_ROW)); + currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), + ReOrgOp.RESHAPE, tmpMatrix); + break; + + case SQL: + currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), + OpOpData.SQLREAD, paramHops); + break; + + case FEDERATED: + currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), + OpOpData.FEDERATED, paramHops); + break; + + default: + throw new ParseException( + source.printErrorLocation() + "processDataExpression():: Unknown operation: " + + source.getOpCode()); } //set identifier meta data (incl dimensions and blocksizes) setIdentifierParams(currBuiltinOp, source.getOutput()); - if( source.getOpCode()==DataExpression.DataOp.READ ) - ((DataOp)currBuiltinOp).setInputBlocksize(target.getBlocksize()); - else if ( source.getOpCode() == DataExpression.DataOp.WRITE ) { - if( source.getVarParam(DataExpression.ROWBLOCKCOUNTPARAM) != null ) - currBuiltinOp.setBlocksize(Integer.parseInt( - source.getVarParam(DataExpression.ROWBLOCKCOUNTPARAM).toString())); + if(source.getOpCode() == DataExpression.DataOp.READ) + ((DataOp) currBuiltinOp).setInputBlocksize(target.getBlocksize()); + else if(source.getOpCode() == DataExpression.DataOp.WRITE) { + if(source.getVarParam(DataExpression.ROWBLOCKCOUNTPARAM) != null) + currBuiltinOp.setBlocksize( + Integer.parseInt(source.getVarParam(DataExpression.ROWBLOCKCOUNTPARAM).toString())); } currBuiltinOp.setParseInfo(source); @@ -2202,25 +2237,25 @@ else if ( source.getOpCode() == DataExpression.DataOp.WRITE ) { } /** - * Construct HOps from parse tree: process BuiltinFunction Expressions in - * MultiAssignment Statements. For all other builtin function expressions, + * Construct HOps from parse tree: process BuiltinFunction Expressions in MultiAssignment Statements. For all other + * builtin function expressions, * processBuiltinFunctionExpression() is used. - * - * @param source built-in function expression + * + * @param source built-in function expression * @param targetList list of data identifiers - * @param hops map of high-level operators + * @param hops map of high-level operators * @return high-level operator */ - private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpression source, ArrayList targetList, - HashMap hops) { + private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpression source, + ArrayList targetList, HashMap hops) { // Construct Hops for all inputs ArrayList inputs = new ArrayList<>(); - inputs.add( processExpression(source.getFirstExpr(), null, hops) ); + inputs.add(processExpression(source.getFirstExpr(), null, hops)); Expression[] expr = source.getAllExpr(); if(expr != null && expr.length > 1) { for(int i = 1; i < expr.length; i++) { - inputs.add( processExpression(expr[i], null, hops) ); + inputs.add(processExpression(expr[i], null, hops)); } } @@ -2233,7 +2268,7 @@ private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpres // Construct Hop for current builtin function expression based on its type Hop currBuiltinOp = null; - switch (source.getOpCode()) { + switch(source.getOpCode()) { case QR: case LU: case EIGEN: @@ -2251,34 +2286,41 @@ private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpres case RCM: // Number of outputs = size of targetList = #of identifiers in source.getOutputs - String[] outputNames = new String[targetList.size()]; - for ( int i=0; i < targetList.size(); i++ ) { + String[] outputNames = new String[targetList.size()]; + for(int i = 0; i < targetList.size(); i++) { outputNames[i] = targetList.get(i).getName(); - Hop output = new DataOp(outputNames[i], DataType.MATRIX, ValueType.FP64, inputs.get(0), OpOpData.FUNCTIONOUTPUT, inputs.get(0).getFilename()); + Hop output = new DataOp(outputNames[i], DataType.MATRIX, ValueType.FP64, inputs.get(0), + OpOpData.FUNCTIONOUTPUT, inputs.get(0).getFilename()); outputs.add(output); } // Create the hop for current function call - currBuiltinOp = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), null, inputs, outputNames, outputs); + currBuiltinOp = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), null, inputs, + outputNames, outputs); break; case COMPRESS: // Number of outputs = size of targetList = #of identifiers in source.getOutputs String[] outputNamesCompress = new String[targetList.size()]; outputNamesCompress[0] = targetList.get(0).getName(); outputNamesCompress[1] = targetList.get(1).getName(); - outputs.add(new DataOp(outputNamesCompress[0], DataType.MATRIX, ValueType.FP64, inputs.get(0), OpOpData.FUNCTIONOUTPUT, inputs.get(0).getFilename())); - outputs.add(new DataOp(outputNamesCompress[1], DataType.FRAME, ValueType.STRING, inputs.get(0), OpOpData.FUNCTIONOUTPUT, inputs.get(0).getFilename())); + outputs.add(new DataOp(outputNamesCompress[0], DataType.MATRIX, ValueType.FP64, inputs.get(0), + OpOpData.FUNCTIONOUTPUT, inputs.get(0).getFilename())); + outputs.add(new DataOp(outputNamesCompress[1], DataType.FRAME, ValueType.STRING, inputs.get(0), + OpOpData.FUNCTIONOUTPUT, inputs.get(0).getFilename())); // Create the hop for current function call - currBuiltinOp = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), null, inputs, outputNamesCompress, outputs); + currBuiltinOp = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), null, inputs, + outputNamesCompress, outputs); break; default: - throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnBuiltinFunctionExpression(): " + source.getOpCode()); + throw new ParseException( + "Invaid Opcode in DMLTranslator:processMultipleReturnBuiltinFunctionExpression(): " + + source.getOpCode()); } // set properties for created hops based on outputs of source expression - for ( int i=0; i < source.getOutputs().length; i++ ) { - setIdentifierParams( outputs.get(i), source.getOutputs()[i]); + for(int i = 0; i < source.getOutputs().length; i++) { + setIdentifierParams(outputs.get(i), source.getOutputs()[i]); outputs.get(i).setParseInfo(source); } currBuiltinOp.setParseInfo(source); @@ -2287,26 +2329,25 @@ private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpres } /** - * Construct Hops from parse tree : Process BuiltinFunction Expression in an - * assignment statement - * + * Construct Hops from parse tree : Process BuiltinFunction Expression in an assignment statement + * * @param source built-in function expression * @param target data identifier - * @param hops map of high-level operators + * @param hops map of high-level operators * @return high-level operator */ private Hop processBuiltinFunctionExpression(BuiltinFunctionExpression source, DataIdentifier target, - HashMap hops) { + HashMap hops) { Hop expr = null; - if(source.getFirstExpr() != null){ + if(source.getFirstExpr() != null) { expr = processExpression(source.getFirstExpr(), null, hops); } Hop expr2 = null; - if (source.getSecondExpr() != null) { + if(source.getSecondExpr() != null) { expr2 = processExpression(source.getSecondExpr(), null, hops); } Hop expr3 = null; - if (source.getThirdExpr() != null) { + if(source.getThirdExpr() != null) { expr3 = processExpression(source.getThirdExpr(), null, hops); } @@ -2314,525 +2355,554 @@ private Hop processBuiltinFunctionExpression(BuiltinFunctionExpression source, D target = (target == null) ? createTarget(source) : target; // Construct the hop based on the type of Builtin function - switch (source.getOpCode()) { - - case EVAL: - case EVALLIST: - currBuiltinOp = new NaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOpN.EVAL, processAllExpressions(source.getAllExpr(), hops)); - break; - - case COLSUM: - case COLMAX: - case COLMIN: - case COLMEAN: - case COLPROD: - case COLVAR: - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), - AggOp.valueOf(source.getOpCode().name().substring(3)), Direction.Col, expr); - break; - - case COLSD: - // colStdDevs = sqrt(colVariances) - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, - target.getValueType(), AggOp.VAR, Direction.Col, expr); - currBuiltinOp = new UnaryOp(target.getName(), DataType.MATRIX, - target.getValueType(), OpOp1.SQRT, currBuiltinOp); - break; - - case ROWSUM: - case ROWMIN: - case ROWMAX: - case ROWMEAN: - case ROWPROD: - case ROWVAR: - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), - AggOp.valueOf(source.getOpCode().name().substring(3)), Direction.Row, expr); - break; - - case ROWINDEXMAX: - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), AggOp.MAXINDEX, + switch(source.getOpCode()) { + + case EVAL: + case EVALLIST: + currBuiltinOp = new NaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOpN.EVAL, + processAllExpressions(source.getAllExpr(), hops)); + break; + + case COLSUM: + case COLMAX: + case COLMIN: + case COLMEAN: + case COLPROD: + case COLVAR: + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), + AggOp.valueOf(source.getOpCode().name().substring(3)), Direction.Col, expr); + break; + + case COLSD: + // colStdDevs = sqrt(colVariances) + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), AggOp.VAR, + Direction.Col, expr); + currBuiltinOp = new UnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), OpOp1.SQRT, + currBuiltinOp); + break; + + case ROWSUM: + case ROWMIN: + case ROWMAX: + case ROWMEAN: + case ROWPROD: + case ROWVAR: + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), + AggOp.valueOf(source.getOpCode().name().substring(3)), Direction.Row, expr); + break; + + case ROWINDEXMAX: + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), AggOp.MAXINDEX, Direction.Row, expr); - break; + break; - case ROWINDEXMIN: - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), AggOp.MININDEX, + case ROWINDEXMIN: + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), AggOp.MININDEX, Direction.Row, expr); - break; - - case ROWSD: - // rowStdDevs = sqrt(rowVariances) - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, - target.getValueType(), AggOp.VAR, Direction.Row, expr); - currBuiltinOp = new UnaryOp(target.getName(), DataType.MATRIX, - target.getValueType(), OpOp1.SQRT, currBuiltinOp); - break; - - case NROW: - // If the dimensions are available at compile time, then create a LiteralOp (constant propagation) - // Else create a UnaryOp so that a control program instruction is generated - currBuiltinOp = (expr.getDim1()==-1) ? new UnaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp1.NROW, expr) : new LiteralOp(expr.getDim1()); - break; - case NCOL: - // If the dimensions are available at compile time, then create a LiteralOp (constant propagation) - // Else create a UnaryOp so that a control program instruction is generated - currBuiltinOp = (expr.getDim2()==-1) ? new UnaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp1.NCOL, expr) : new LiteralOp(expr.getDim2()); - break; - case LENGTH: - // If the dimensions are available at compile time, then create a LiteralOp (constant propagation) - // Else create a UnaryOp so that a control program instruction is generated - currBuiltinOp = (expr.getDim1()==-1 || expr.getDim2()==-1) ? new UnaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp1.LENGTH, expr) : new LiteralOp(expr.getDim1()*expr.getDim2()); - break; - - case LINEAGE: - //construct hop and enable lineage tracing if necessary - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp1.LINEAGE, expr); - DMLScript.LINEAGE = true; - break; - - case LIST: - currBuiltinOp = new NaryOp(target.getName(), DataType.LIST, ValueType.UNKNOWN, - OpOpN.LIST, processAllExpressions(source.getAllExpr(), hops)); - break; - - case EXISTS: - currBuiltinOp = new UnaryOp(target.getName(), DataType.SCALAR, - target.getValueType(), OpOp1.EXISTS, expr); - break; - - case SUM: - case PROD: - case VAR: - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), - AggOp.valueOf(source.getOpCode().name()), Direction.RowCol, expr); - break; + break; - case MEAN: - if ( expr2 == null ) { - // example: x = mean(Y); - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), AggOp.MEAN, - Direction.RowCol, expr); - } - else { - // example: x = mean(Y,W); - // stable weighted mean is implemented by using centralMoment with order = 0 - Hop orderHop = new LiteralOp(0); - currBuiltinOp=new TernaryOp(target.getName(), DataType.SCALAR, - target.getValueType(), OpOp3.MOMENT, expr, expr2, orderHop); - } - break; - - case SD: - // stdDev = sqrt(variance) - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.SCALAR, - target.getValueType(), AggOp.VAR, Direction.RowCol, expr); - HopRewriteUtils.setOutputParametersForScalar(currBuiltinOp); - currBuiltinOp = new UnaryOp(target.getName(), DataType.SCALAR, - target.getValueType(), OpOp1.SQRT, currBuiltinOp); - break; - - case MIN: - case MAX: - //construct AggUnary for min(X) but BinaryOp for min(X,Y) and NaryOp for min(X,Y,Z) - currBuiltinOp = (expr2 == null) ? - new AggUnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), - AggOp.valueOf(source.getOpCode().name()), Direction.RowCol, expr) : - (source.getAllExpr().length == 2) ? - new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOp2.valueOf(source.getOpCode().name()), expr, expr2) : - new NaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOpN.valueOf(source.getOpCode().name()), processAllExpressions(source.getAllExpr(), hops)); - break; - case EINSUM: - currBuiltinOp = new NaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOpN.valueOf(source.getOpCode().name()), processAllExpressions(source.getAllExpr(), hops)); - break; - case PPRED: - String sop = ((StringIdentifier)source.getThirdExpr()).getValue(); - sop = sop.replace("\"", ""); - OpOp2 operation; - if ( sop.equalsIgnoreCase(Opcodes.GREATEREQUAL.toString()) ) - operation = OpOp2.GREATEREQUAL; - else if ( sop.equalsIgnoreCase(Opcodes.GREATER.toString()) ) - operation = OpOp2.GREATER; - else if ( sop.equalsIgnoreCase(Opcodes.LESSEQUAL.toString()) ) - operation = OpOp2.LESSEQUAL; - else if ( sop.equalsIgnoreCase(Opcodes.LESS.toString()) ) - operation = OpOp2.LESS; - else if ( sop.equalsIgnoreCase(Opcodes.EQUAL.toString()) ) - operation = OpOp2.EQUAL; - else if ( sop.equalsIgnoreCase(Opcodes.NOTEQUAL.toString()) ) - operation = OpOp2.NOTEQUAL; - else { - throw new ParseException(source.printErrorLocation() + "Unknown argument (" + sop + ") for PPRED."); - } - currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), operation, expr, expr2); - break; + case ROWSD: + // rowStdDevs = sqrt(rowVariances) + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), AggOp.VAR, + Direction.Row, expr); + currBuiltinOp = new UnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), OpOp1.SQRT, + currBuiltinOp); + break; - case TRACE: - currBuiltinOp = new AggUnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), AggOp.TRACE, - Direction.RowCol, expr); - break; - - case TRANS: - case DIAG: - case REV: - currBuiltinOp = new ReorgOp(target.getName(), DataType.MATRIX, - target.getValueType(), ReOrgOp.valueOf(source.getOpCode().name()), expr); - break; - - case ROLL: - ArrayList inputs = new ArrayList<>(); - inputs.add(expr); - inputs.add(expr2); - currBuiltinOp = new ReorgOp(target.getName(), DataType.MATRIX, - target.getValueType(), ReOrgOp.valueOf(source.getOpCode().name()), inputs); - break; - - case CBIND: - case RBIND: - OpOp2 appendOp2 = (source.getOpCode()==Builtins.CBIND) ? OpOp2.CBIND : OpOp2.RBIND; - OpOpN appendOpN = (source.getOpCode()==Builtins.CBIND) ? OpOpN.CBIND : OpOpN.RBIND; - currBuiltinOp = (source.getAllExpr().length == 2) ? - new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), appendOp2, expr, expr2) : - new NaryOp(target.getName(), target.getDataType(), target.getValueType(), appendOpN, + case NROW: + // If the dimensions are available at compile time, then create a LiteralOp (constant propagation) + // Else create a UnaryOp so that a control program instruction is generated + currBuiltinOp = (expr.getDim1() == -1) ? new UnaryOp(target.getName(), target.getDataType(), + target.getValueType(), OpOp1.NROW, expr) : new LiteralOp(expr.getDim1()); + break; + case NCOL: + // If the dimensions are available at compile time, then create a LiteralOp (constant propagation) + // Else create a UnaryOp so that a control program instruction is generated + currBuiltinOp = (expr.getDim2() == -1) ? new UnaryOp(target.getName(), target.getDataType(), + target.getValueType(), OpOp1.NCOL, expr) : new LiteralOp(expr.getDim2()); + break; + case LENGTH: + // If the dimensions are available at compile time, then create a LiteralOp (constant propagation) + // Else create a UnaryOp so that a control program instruction is generated + currBuiltinOp = (expr.getDim1() == -1 || expr.getDim2() == -1) ? new UnaryOp(target.getName(), + target.getDataType(), target.getValueType(), OpOp1.LENGTH, expr) : new LiteralOp( + expr.getDim1() * expr.getDim2()); + break; + + case LINEAGE: + //construct hop and enable lineage tracing if necessary + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp1.LINEAGE, expr); + DMLScript.LINEAGE = true; + break; + + case LIST: + currBuiltinOp = new NaryOp(target.getName(), DataType.LIST, ValueType.UNKNOWN, OpOpN.LIST, processAllExpressions(source.getAllExpr(), hops)); - break; - - case TABLE: - - // Always a TertiaryOp is created for table(). - // - create a hop for weights, if not provided in the function call. - int numTableArgs = source._args.length; - - switch(numTableArgs) { - case 2: - case 4: - // example DML statement: F = ctable(A,B) or F = ctable(A,B,10,15) - // here, weight is interpreted as 1.0 - Hop weightHop = new LiteralOp(1.0); - // set dimensions - weightHop.setDim1(0); - weightHop.setDim2(0); - weightHop.setNnz(-1); - weightHop.setBlocksize(0); - - if ( numTableArgs == 2 ) - currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp3.CTABLE, expr, expr2, weightHop); + break; + + case EXISTS: + currBuiltinOp = new UnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), OpOp1.EXISTS, + expr); + break; + + case SUM: + case PROD: + case VAR: + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), + AggOp.valueOf(source.getOpCode().name()), Direction.RowCol, expr); + break; + + case MEAN: + if(expr2 == null) { + // example: x = mean(Y); + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), AggOp.MEAN, + Direction.RowCol, expr); + } else { - Hop outDim1 = processExpression(source._args[2], null, hops); - Hop outDim2 = processExpression(source._args[3], null, hops); - currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOp3.CTABLE, expr, expr2, weightHop, outDim1, outDim2, new LiteralOp(true)); + // example: x = mean(Y,W); + // stable weighted mean is implemented by using centralMoment with order = 0 + Hop orderHop = new LiteralOp(0); + currBuiltinOp = new TernaryOp(target.getName(), DataType.SCALAR, target.getValueType(), + OpOp3.MOMENT, expr, expr2, orderHop); } break; - case 3: - case 5: - case 6: - // example DML statement: F = ctable(A,B,W) or F = ctable(A,B,W,10,15) - if (numTableArgs == 3) - currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp3.CTABLE, expr, expr2, expr3); + case SD: + // stdDev = sqrt(variance) + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), AggOp.VAR, + Direction.RowCol, expr); + HopRewriteUtils.setOutputParametersForScalar(currBuiltinOp); + currBuiltinOp = new UnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), OpOp1.SQRT, + currBuiltinOp); + break; + + case MIN: + case MAX: + //construct AggUnary for min(X) but BinaryOp for min(X,Y) and NaryOp for min(X,Y,Z) + currBuiltinOp = (expr2 == null) ? new AggUnaryOp(target.getName(), DataType.SCALAR, + target.getValueType(), AggOp.valueOf(source.getOpCode().name()), Direction.RowCol, expr) : ( + source.getAllExpr().length == 2) ? new BinaryOp(target.getName(), target.getDataType(), + target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), expr, expr2) : new NaryOp( + target.getName(), target.getDataType(), target.getValueType(), + OpOpN.valueOf(source.getOpCode().name()), processAllExpressions(source.getAllExpr(), hops)); + break; + case EINSUM: + currBuiltinOp = new NaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOpN.valueOf(source.getOpCode().name()), processAllExpressions(source.getAllExpr(), hops)); + break; + case PPRED: + String sop = ((StringIdentifier) source.getThirdExpr()).getValue(); + sop = sop.replace("\"", ""); + OpOp2 operation; + if(sop.equalsIgnoreCase(Opcodes.GREATEREQUAL.toString())) + operation = OpOp2.GREATEREQUAL; + else if(sop.equalsIgnoreCase(Opcodes.GREATER.toString())) + operation = OpOp2.GREATER; + else if(sop.equalsIgnoreCase(Opcodes.LESSEQUAL.toString())) + operation = OpOp2.LESSEQUAL; + else if(sop.equalsIgnoreCase(Opcodes.LESS.toString())) + operation = OpOp2.LESS; + else if(sop.equalsIgnoreCase(Opcodes.EQUAL.toString())) + operation = OpOp2.EQUAL; + else if(sop.equalsIgnoreCase(Opcodes.NOTEQUAL.toString())) + operation = OpOp2.NOTEQUAL; else { - Hop outDim1 = processExpression(source._args[3], null, hops); - Hop outDim2 = processExpression(source._args[4], null, hops); - Hop outputEmptyBlocks = numTableArgs == 6 ? - processExpression(source._args[5], null, hops) : new LiteralOp(true); - currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOp3.CTABLE, expr, expr2, expr3, outDim1, outDim2, outputEmptyBlocks); + throw new ParseException(source.printErrorLocation() + "Unknown argument (" + sop + ") for PPRED."); } + currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), operation, + expr, expr2); break; - default: - throw new ParseException("Invalid number of arguments "+ numTableArgs + " to table() function."); - } - break; + case TRACE: + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), AggOp.TRACE, + Direction.RowCol, expr); + break; - //data type casts - case CAST_AS_SCALAR: - currBuiltinOp = new UnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), OpOp1.CAST_AS_SCALAR, expr); - break; - case CAST_AS_MATRIX: - currBuiltinOp = new UnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), OpOp1.CAST_AS_MATRIX, expr); - break; - case CAST_AS_FRAME: - if(expr2 != null) - currBuiltinOp = new BinaryOp(target.getName(), DataType.FRAME, target.getValueType(), OpOp2.CAST_AS_FRAME, expr, expr2); - else - currBuiltinOp = new UnaryOp(target.getName(), DataType.FRAME, target.getValueType(), OpOp1.CAST_AS_FRAME, expr); - break; - case CAST_AS_LIST: - currBuiltinOp = new UnaryOp(target.getName(), DataType.LIST, target.getValueType(), OpOp1.CAST_AS_LIST, expr); - break; - - //value type casts - case CAST_AS_DOUBLE: - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, OpOp1.CAST_AS_DOUBLE, expr); - break; - case CAST_AS_INT: - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.INT64, OpOp1.CAST_AS_INT, expr); - break; - case CAST_AS_BOOLEAN: - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.BOOLEAN, OpOp1.CAST_AS_BOOLEAN, expr); - break; - case LOCAL: - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, OpOp1.LOCAL, expr); - break; - case COMPRESS: - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, OpOp1.COMPRESS, expr); - break; - case DECOMPRESS: - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, OpOp1.DECOMPRESS, expr); - break; - case QUANTIZE_COMPRESS: - currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), expr, expr2); - break; - - // Boolean binary - case XOR: - case BITWAND: - case BITWOR: - case BITWXOR: - case BITWSHIFTL: - case BITWSHIFTR: - currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), expr, expr2); - break; - case ABS: - case SIN: - case COS: - case TAN: - case ASIN: - case ACOS: - case ATAN: - case SINH: - case COSH: - case TANH: - case SIGN: - case SQRT: - case EXP: - case ROUND: - case CEIL: - case FLOOR: - case CUMSUM: - case ROWCUMSUM: - case CUMPROD: - case CUMSUMPROD: - case CUMMIN: - case CUMMAX: - case ISNA: - case ISNAN: - case ISINF: - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOp1.valueOf(source.getOpCode().name()), expr); - break; - case DROP_INVALID_TYPE: - case DROP_INVALID_LENGTH: - case VALUE_SWAP: - case FRAME_ROW_REPLICATE: - case APPLY_SCHEMA: - currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), expr, expr2); - break; - case MAP: - currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp3.valueOf(source.getOpCode().name()), - expr, expr2, (expr3==null) ? new LiteralOp(0L) : expr3); - break; - - case LOG: - if (expr2 == null) { - OpOp1 mathOp2; - switch (source.getOpCode()) { - case LOG: - mathOp2 = OpOp1.LOG; + case TRANS: + case DIAG: + case REV: + currBuiltinOp = new ReorgOp(target.getName(), DataType.MATRIX, target.getValueType(), + ReOrgOp.valueOf(source.getOpCode().name()), expr); + break; + + case ROLL: + ArrayList inputs = new ArrayList<>(); + inputs.add(expr); + inputs.add(expr2); + currBuiltinOp = new ReorgOp(target.getName(), DataType.MATRIX, target.getValueType(), + ReOrgOp.valueOf(source.getOpCode().name()), inputs); + break; + + case CBIND: + case RBIND: + OpOp2 appendOp2 = (source.getOpCode() == Builtins.CBIND) ? OpOp2.CBIND : OpOp2.RBIND; + OpOpN appendOpN = (source.getOpCode() == Builtins.CBIND) ? OpOpN.CBIND : OpOpN.RBIND; + currBuiltinOp = (source.getAllExpr().length == 2) ? new BinaryOp(target.getName(), target.getDataType(), + target.getValueType(), appendOp2, expr, expr2) : new NaryOp(target.getName(), target.getDataType(), + target.getValueType(), appendOpN, processAllExpressions(source.getAllExpr(), hops)); + break; + + case TABLE: + + // Always a TertiaryOp is created for table(). + // - create a hop for weights, if not provided in the function call. + int numTableArgs = source._args.length; + + switch(numTableArgs) { + case 2: + case 4: + // example DML statement: F = ctable(A,B) or F = ctable(A,B,10,15) + // here, weight is interpreted as 1.0 + Hop weightHop = new LiteralOp(1.0); + // set dimensions + weightHop.setDim1(0); + weightHop.setDim2(0); + weightHop.setNnz(-1); + weightHop.setBlocksize(0); + + if(numTableArgs == 2) + currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp3.CTABLE, expr, expr2, weightHop); + else { + Hop outDim1 = processExpression(source._args[2], null, hops); + Hop outDim2 = processExpression(source._args[3], null, hops); + currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp3.CTABLE, expr, expr2, weightHop, outDim1, outDim2, new LiteralOp(true)); + } + break; + + case 3: + case 5: + case 6: + // example DML statement: F = ctable(A,B,W) or F = ctable(A,B,W,10,15) + if(numTableArgs == 3) + currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp3.CTABLE, expr, expr2, expr3); + else { + Hop outDim1 = processExpression(source._args[3], null, hops); + Hop outDim2 = processExpression(source._args[4], null, hops); + Hop outputEmptyBlocks = + numTableArgs == 6 ? processExpression(source._args[5], null, hops) : new LiteralOp( + true); + currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp3.CTABLE, expr, expr2, expr3, outDim1, outDim2, outputEmptyBlocks); + } break; + default: - throw new ParseException(source.printErrorLocation() + - "processBuiltinFunctionExpression():: Could not find Operation type for builtin function: " - + source.getOpCode()); + throw new ParseException( + "Invalid number of arguments " + numTableArgs + " to table() function."); + } + break; + + //data type casts + case CAST_AS_SCALAR: + currBuiltinOp = new UnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), + OpOp1.CAST_AS_SCALAR, expr); + break; + case CAST_AS_MATRIX: + currBuiltinOp = new UnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOp1.CAST_AS_MATRIX, expr); + break; + case CAST_AS_FRAME: + if(expr2 != null) + currBuiltinOp = new BinaryOp(target.getName(), DataType.FRAME, target.getValueType(), + OpOp2.CAST_AS_FRAME, expr, expr2); + else + currBuiltinOp = new UnaryOp(target.getName(), DataType.FRAME, target.getValueType(), + OpOp1.CAST_AS_FRAME, expr); + break; + case CAST_AS_LIST: + currBuiltinOp = new UnaryOp(target.getName(), DataType.LIST, target.getValueType(), OpOp1.CAST_AS_LIST, + expr); + break; + + //value type casts + case CAST_AS_DOUBLE: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, + OpOp1.CAST_AS_DOUBLE, expr); + break; + case CAST_AS_INT: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.INT64, OpOp1.CAST_AS_INT, + expr); + break; + case CAST_AS_BOOLEAN: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.BOOLEAN, + OpOp1.CAST_AS_BOOLEAN, expr); + break; + case LOCAL: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, OpOp1.LOCAL, expr); + break; + case COMPRESS: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, OpOp1.COMPRESS, + expr); + break; + case DECOMPRESS: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, OpOp1.DECOMPRESS, + expr); + break; + case QUANTIZE_COMPRESS: + currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp2.valueOf(source.getOpCode().name()), expr, expr2); + break; + + // Boolean binary + case XOR: + case BITWAND: + case BITWOR: + case BITWXOR: + case BITWSHIFTL: + case BITWSHIFTR: + currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp2.valueOf(source.getOpCode().name()), expr, expr2); + break; + case ABS: + case SIN: + case COS: + case TAN: + case ASIN: + case ACOS: + case ATAN: + case SINH: + case COSH: + case TANH: + case SIGN: + case SQRT: + case EXP: + case ROUND: + case CEIL: + case FLOOR: + case CUMSUM: + case ROWCUMSUM: + case CUMPROD: + case CUMSUMPROD: + case CUMMIN: + case CUMMAX: + case ISNA: + case ISNAN: + case ISINF: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp1.valueOf(source.getOpCode().name()), expr); + break; + case DROP_INVALID_TYPE: + case DROP_INVALID_LENGTH: + case VALUE_SWAP: + case FRAME_ROW_REPLICATE: + case APPLY_SCHEMA: + currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp2.valueOf(source.getOpCode().name()), expr, expr2); + break; + case MAP: + currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp3.valueOf(source.getOpCode().name()), expr, expr2, (expr3 == null) ? new LiteralOp(0L) : expr3); + break; + + case LOG: + if(expr2 == null) { + OpOp1 mathOp2; + switch(source.getOpCode()) { + case LOG: + mathOp2 = OpOp1.LOG; + break; + default: + throw new ParseException(source.printErrorLocation() + + "processBuiltinFunctionExpression():: Could not find Operation type for builtin function: " + + source.getOpCode()); } - currBuiltinOp = new UnaryOp(target.getName(), - target.getDataType(), target.getValueType(), mathOp2, expr); - } else { + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), mathOp2, + expr); + } + else { OpOp2 mathOp3; - switch (source.getOpCode()) { - case LOG: - mathOp3 = OpOp2.LOG; - break; - default: - throw new ParseException(source.printErrorLocation() + - "processBuiltinFunctionExpression():: Could not find Operation type for builtin function: " - + source.getOpCode()); + switch(source.getOpCode()) { + case LOG: + mathOp3 = OpOp2.LOG; + break; + default: + throw new ParseException(source.printErrorLocation() + + "processBuiltinFunctionExpression():: Could not find Operation type for builtin function: " + + source.getOpCode()); } currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), mathOp3, - expr, expr2); + expr, expr2); } - break; - - case MOMENT: - case COV: - case QUANTILE: - case INTERQUANTILE: - currBuiltinOp = (expr3 == null) ? new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOp2.valueOf(source.getOpCode().name()), expr, expr2) : new TernaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp3.valueOf(source.getOpCode().name()), expr, expr2,expr3); - break; - - case IQM: - case MEDIAN: - currBuiltinOp = (expr2 == null) ? new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), - OpOp1.valueOf(source.getOpCode().name()), expr) : new BinaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), expr, expr2); - break; - - case IFELSE: - currBuiltinOp=new TernaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp3.IFELSE, expr, expr2, expr3); - break; - - case SEQ: - HashMap randParams = new HashMap<>(); - randParams.put(Statement.SEQ_FROM, expr); - randParams.put(Statement.SEQ_TO, expr2); - randParams.put(Statement.SEQ_INCR, (expr3!=null)?expr3 : new LiteralOp(1)); - //note incr: default -1 (for from>to) handled during runtime - currBuiltinOp = new DataGenOp(OpOpDG.SEQ, target, randParams); - break; - - case TIME: - currBuiltinOp = new DataGenOp(OpOpDG.TIME, target); - break; - - case SAMPLE: - { - Expression[] in = source.getAllExpr(); - - // arguments: range/size/replace/seed; defaults: replace=FALSE - - HashMap tmpparams = new HashMap<>(); - tmpparams.put(DataExpression.RAND_MAX, expr); //range - tmpparams.put(DataExpression.RAND_ROWS, expr2); - tmpparams.put(DataExpression.RAND_COLS, new LiteralOp(1)); - - if ( in.length == 4 ) - { - tmpparams.put(DataExpression.RAND_PDF, expr3); - Hop seed = processExpression(in[3], null, hops); - tmpparams.put(DataExpression.RAND_SEED, seed); - } - else if ( in.length == 3 ) - { - // check if the third argument is "replace" or "seed" - if ( expr3.getValueType() == ValueType.BOOLEAN ) - { + break; + + case MOMENT: + case COV: + case QUANTILE: + case INTERQUANTILE: + currBuiltinOp = (expr3 == null) ? new BinaryOp(target.getName(), target.getDataType(), + target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), expr, expr2) : new TernaryOp( + target.getName(), target.getDataType(), target.getValueType(), + OpOp3.valueOf(source.getOpCode().name()), expr, expr2, expr3); + break; + + case IQM: + case MEDIAN: + currBuiltinOp = (expr2 == null) ? new UnaryOp(target.getName(), target.getDataType(), + target.getValueType(), OpOp1.valueOf(source.getOpCode().name()), expr) : new BinaryOp( + target.getName(), target.getDataType(), target.getValueType(), + OpOp2.valueOf(source.getOpCode().name()), expr, expr2); + break; + + case IFELSE: + currBuiltinOp = new TernaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp3.IFELSE, expr, expr2, expr3); + break; + + case SEQ: + HashMap randParams = new HashMap<>(); + randParams.put(Statement.SEQ_FROM, expr); + randParams.put(Statement.SEQ_TO, expr2); + randParams.put(Statement.SEQ_INCR, (expr3 != null) ? expr3 : new LiteralOp(1)); + //note incr: default -1 (for from>to) handled during runtime + currBuiltinOp = new DataGenOp(OpOpDG.SEQ, target, randParams); + break; + + case TIME: + currBuiltinOp = new DataGenOp(OpOpDG.TIME, target); + break; + + case SAMPLE: { + Expression[] in = source.getAllExpr(); + + // arguments: range/size/replace/seed; defaults: replace=FALSE + + HashMap tmpparams = new HashMap<>(); + tmpparams.put(DataExpression.RAND_MAX, expr); //range + tmpparams.put(DataExpression.RAND_ROWS, expr2); + tmpparams.put(DataExpression.RAND_COLS, new LiteralOp(1)); + + if(in.length == 4) { tmpparams.put(DataExpression.RAND_PDF, expr3); - tmpparams.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED) ); + Hop seed = processExpression(in[3], null, hops); + tmpparams.put(DataExpression.RAND_SEED, seed); } - else if ( expr3.getValueType() == ValueType.INT64 ) - { + else if(in.length == 3) { + // check if the third argument is "replace" or "seed" + if(expr3.getValueType() == ValueType.BOOLEAN) { + tmpparams.put(DataExpression.RAND_PDF, expr3); + tmpparams.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED)); + } + else if(expr3.getValueType() == ValueType.INT64) { + tmpparams.put(DataExpression.RAND_PDF, new LiteralOp(false)); + tmpparams.put(DataExpression.RAND_SEED, expr3); + } + else + throw new HopsException("Invalid input type " + expr3.getValueType() + " in sample()."); + + } + else if(in.length == 2) { tmpparams.put(DataExpression.RAND_PDF, new LiteralOp(false)); - tmpparams.put(DataExpression.RAND_SEED, expr3 ); + tmpparams.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED)); } - else - throw new HopsException("Invalid input type " + expr3.getValueType() + " in sample()."); - - } - else if ( in.length == 2 ) - { - tmpparams.put(DataExpression.RAND_PDF, new LiteralOp(false)); - tmpparams.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED) ); - } - - currBuiltinOp = new DataGenOp(OpOpDG.SAMPLE, target, tmpparams); - break; - } - - case SOLVE: - currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.SOLVE, expr, expr2); - break; - - case INVERSE: - case SQRT_MATRIX_JAVA: - case CHOLESKY: - case TYPEOF: - case DET: - case DETECTSCHEMA: - case COLNAMES: - currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), - target.getValueType(), OpOp1.valueOf(source.getOpCode().name()), expr); - break; - - case OUTER: - if( !(expr3 instanceof LiteralOp) ) - throw new HopsException("Operator for outer builtin function must be a constant: "+expr3); - OpOp2 op = OpOp2.valueOfByOpcode(((LiteralOp)expr3).getStringValue()); - if( op == null ) - throw new HopsException("Unsupported outer vector binary operation: "+((LiteralOp)expr3).getStringValue()); - - currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, target.getValueType(), op, expr, expr2, true); - currBuiltinOp.refreshSizeInformation(); //force size reevaluation according to 'outer' flag otherwise danger of incorrect dims - break; - - case BIASADD: - case BIASMULT: { - ArrayList inHops1 = new ArrayList<>(); - inHops1.add(expr); - inHops1.add(expr2); - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), inHops1); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL: - case MAX_POOL: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForPoolingForwardIM2COL(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL_BACKWARD: - case MAX_POOL_BACKWARD: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOpPoolingCOL2IM(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case CONV2D: - case CONV2D_BACKWARD_FILTER: - case CONV2D_BACKWARD_DATA: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOp(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - - case ROW_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Row, expr); - break; - - case COL_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Col, expr); - break; - - case GET_CATEGORICAL_MASK: - currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, OpOp2.GET_CATEGORICAL_MASK, expr, expr2); - break; - default: - throw new ParseException("Unsupported builtin function type: "+source.getOpCode()); - } - - boolean isConvolution = source.getOpCode() == Builtins.CONV2D || source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || - source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || - source.getOpCode() == Builtins.MAX_POOL || source.getOpCode() == Builtins.MAX_POOL_BACKWARD || - source.getOpCode() == Builtins.AVG_POOL || source.getOpCode() == Builtins.AVG_POOL_BACKWARD; - if( !isConvolution) { + + currBuiltinOp = new DataGenOp(OpOpDG.SAMPLE, target, tmpparams); + break; + } + + case SOLVE: + currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.SOLVE, + expr, expr2); + break; + + case INVERSE: + case SQRT_MATRIX_JAVA: + case CHOLESKY: + case TYPEOF: + case DET: + case DETECTSCHEMA: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp1.valueOf(source.getOpCode().name()), expr); + break; + + case SET_NAMES: + currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp2.SET_COLNAMES, expr, expr2); + break; + + case GET_NAMES: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp1.COLNAMES, expr); + break; + case COLNAMES: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), + OpOp1.valueOf(source.getOpCode().name()), expr); + break; + + case OUTER: + if(!(expr3 instanceof LiteralOp)) + throw new HopsException("Operator for outer builtin function must be a constant: " + expr3); + OpOp2 op = OpOp2.valueOfByOpcode(((LiteralOp) expr3).getStringValue()); + if(op == null) + throw new HopsException( + "Unsupported outer vector binary operation: " + ((LiteralOp) expr3).getStringValue()); + + currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, target.getValueType(), op, expr, expr2, + true); + currBuiltinOp.refreshSizeInformation(); //force size reevaluation according to 'outer' flag otherwise danger of incorrect dims + break; + + case BIASADD: + case BIASMULT: { + ArrayList inHops1 = new ArrayList<>(); + inHops1.add(expr); + inHops1.add(expr2); + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), inHops1); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case AVG_POOL: + case MAX_POOL: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), + getALHopsForPoolingForwardIM2COL(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case AVG_POOL_BACKWARD: + case MAX_POOL_BACKWARD: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOpPoolingCOL2IM(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case CONV2D: + case CONV2D_BACKWARD_FILTER: + case CONV2D_BACKWARD_DATA: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOp(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + + case ROW_COUNT_DISTINCT: + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), + AggOp.COUNT_DISTINCT, Direction.Row, expr); + break; + + case COL_COUNT_DISTINCT: + currBuiltinOp = new AggUnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), + AggOp.COUNT_DISTINCT, Direction.Col, expr); + break; + + case GET_CATEGORICAL_MASK: + currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, + OpOp2.GET_CATEGORICAL_MASK, expr, expr2); + break; + default: + throw new ParseException("Unsupported builtin function type: " + source.getOpCode()); + } + + boolean isConvolution = + source.getOpCode() == Builtins.CONV2D || source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || + source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || source.getOpCode() == Builtins.MAX_POOL || + source.getOpCode() == Builtins.MAX_POOL_BACKWARD || source.getOpCode() == Builtins.AVG_POOL || + source.getOpCode() == Builtins.AVG_POOL_BACKWARD; + if(!isConvolution) { // Since the dimension of output doesnot match that of input variable for these operations setIdentifierParams(currBuiltinOp, source.getOutput()); } @@ -2842,7 +2912,7 @@ else if ( in.length == 2 ) private Hop[] processAllExpressions(Expression[] expr, HashMap hops) { Hop[] ret = new Hop[expr.length]; - for(int i=0; i getALHopsForConvOpPoolingCOL2IM(Hop first, BuiltinFunctionExpression source, int skip, HashMap hops) { + private ArrayList getALHopsForConvOpPoolingCOL2IM(Hop first, BuiltinFunctionExpression source, int skip, + HashMap hops) { ArrayList ret = new ArrayList<>(); ret.add(first); Expression[] allExpr = source.getAllExpr(); for(int i = skip; i < allExpr.length; i++) { if(i == 11) { - ret.add(processExpression(allExpr[7], null, hops)); // Make number of channels of images and filter the same + ret.add( + processExpression(allExpr[7], null, hops)); // Make number of channels of images and filter the same } else ret.add(processExpression(allExpr[i], null, hops)); @@ -2868,7 +2940,8 @@ private ArrayList getALHopsForConvOpPoolingCOL2IM(Hop first, BuiltinFunctio return ret; } - private ArrayList getALHopsForPoolingForwardIM2COL(Hop first, BuiltinFunctionExpression source, int skip, HashMap hops) { + private ArrayList getALHopsForPoolingForwardIM2COL(Hop first, BuiltinFunctionExpression source, int skip, + HashMap hops) { ArrayList ret = new ArrayList<>(); ret.add(first); Expression[] allExpr = source.getAllExpr(); @@ -2879,7 +2952,7 @@ private ArrayList getALHopsForPoolingForwardIM2COL(Hop first, BuiltinFuncti Expression numChannels = allExpr[6]; for(int i = skip; i < allExpr.length; i++) { - if(i == 10) { + if(i == 10) { ret.add(processExpression(numChannels, null, hops)); } else @@ -2889,7 +2962,8 @@ private ArrayList getALHopsForPoolingForwardIM2COL(Hop first, BuiltinFuncti } @SuppressWarnings("unused") //TODO remove if not used - private ArrayList getALHopsForConvOpPoolingIM2COL(Hop first, BuiltinFunctionExpression source, int skip, HashMap hops) { + private ArrayList getALHopsForConvOpPoolingIM2COL(Hop first, BuiltinFunctionExpression source, int skip, + HashMap hops) { ArrayList ret = new ArrayList(); ret.add(first); Expression[] allExpr = source.getAllExpr(); @@ -2904,8 +2978,8 @@ else if(skip == 2) { throw new ParseException("Unsupported skip"); } - for (int i = skip; i < allExpr.length; i++) { - if (i == numImgIndex) { // skip=1 ==> i==5 and skip=2 => i==6 + for(int i = skip; i < allExpr.length; i++) { + if(i == numImgIndex) { // skip=1 ==> i==5 and skip=2 => i==6 Expression numImg = allExpr[numImgIndex]; Expression numChannels = allExpr[numImgIndex + 1]; BinaryExpression tmp = new BinaryExpression(org.apache.sysds.parser.Expression.BinaryOp.MULT, numImg); @@ -2914,13 +2988,15 @@ else if(skip == 2) { ret.add(processTempIntExpression(tmp, hops)); ret.add(processExpression(new IntIdentifier(1, numImg), null, hops)); i++; - } else + } + else ret.add(processExpression(allExpr[i], null, hops)); } return ret; } - private ArrayList getALHopsForConvOp(Hop first, BuiltinFunctionExpression source, int skip, HashMap hops) { + private ArrayList getALHopsForConvOp(Hop first, BuiltinFunctionExpression source, int skip, + HashMap hops) { ArrayList ret = new ArrayList<>(); ret.add(first); Expression[] allExpr = source.getAllExpr(); @@ -2931,16 +3007,16 @@ private ArrayList getALHopsForConvOp(Hop first, BuiltinFunctionExpression s } public void setIdentifierParams(Hop h, Identifier id) { - if( id.getDim1()>= 0 ) + if(id.getDim1() >= 0) h.setDim1(id.getDim1()); - if( id.getDim2()>= 0 ) + if(id.getDim2() >= 0) h.setDim2(id.getDim2()); - if( id.getNnz()>= 0 ) + if(id.getNnz() >= 0) h.setNnz(id.getNnz()); h.setBlocksize(id.getBlocksize()); } - private boolean prepareReadAfterWrite( DMLProgram prog, HashMap pWrites ) { + private boolean prepareReadAfterWrite(DMLProgram prog, HashMap pWrites) { boolean ret = false; //process functions @@ -2950,81 +3026,74 @@ private boolean prepareReadAfterWrite( DMLProgram prog, HashMap pWrites ) - { + private boolean prepareReadAfterWrite(StatementBlock sb, HashMap pWrites) { boolean ret = false; - if(sb instanceof FunctionStatementBlock) - { + if(sb instanceof FunctionStatementBlock) { FunctionStatementBlock fsb = (FunctionStatementBlock) sb; - FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0); - for (StatementBlock csb : fstmt.getBody()) + FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0); + for(StatementBlock csb : fstmt.getBody()) ret |= prepareReadAfterWrite(csb, pWrites); } - else if(sb instanceof WhileStatementBlock) - { + else if(sb instanceof WhileStatementBlock) { WhileStatementBlock wsb = (WhileStatementBlock) sb; - WhileStatement wstmt = (WhileStatement)wsb.getStatement(0); - for (StatementBlock csb : wstmt.getBody()) + WhileStatement wstmt = (WhileStatement) wsb.getStatement(0); + for(StatementBlock csb : wstmt.getBody()) ret |= prepareReadAfterWrite(csb, pWrites); } - else if(sb instanceof IfStatementBlock) - { + else if(sb instanceof IfStatementBlock) { IfStatementBlock isb = (IfStatementBlock) sb; - IfStatement istmt = (IfStatement)isb.getStatement(0); - for (StatementBlock csb : istmt.getIfBody()) + IfStatement istmt = (IfStatement) isb.getStatement(0); + for(StatementBlock csb : istmt.getIfBody()) ret |= prepareReadAfterWrite(csb, pWrites); - for (StatementBlock csb : istmt.getElseBody()) + for(StatementBlock csb : istmt.getElseBody()) ret |= prepareReadAfterWrite(csb, pWrites); } else if(sb instanceof ForStatementBlock) //incl parfor { ForStatementBlock fsb = (ForStatementBlock) sb; - ForStatement fstmt = (ForStatement)fsb.getStatement(0); - for (StatementBlock csb : fstmt.getBody()) + ForStatement fstmt = (ForStatement) fsb.getStatement(0); + for(StatementBlock csb : fstmt.getBody()) ret |= prepareReadAfterWrite(csb, pWrites); } else //generic (last-level) { - for( Statement s : sb.getStatements() ) - { + for(Statement s : sb.getStatements()) { //collect persistent write information - if( s instanceof OutputStatement ) - { + if(s instanceof OutputStatement) { OutputStatement os = (OutputStatement) s; String pfname = os.getExprParam(DataExpression.IO_FILENAME).toString(); DataIdentifier di = (DataIdentifier) os.getSource().getOutput(); pWrites.put(pfname, di); } //propagate size information into reads-after-write - else if( s instanceof AssignmentStatement - && ((AssignmentStatement)s).getSource() instanceof DataExpression ) - { - DataExpression dexpr = (DataExpression) ((AssignmentStatement)s).getSource(); - if (dexpr.isRead()) { + else if(s instanceof AssignmentStatement && + ((AssignmentStatement) s).getSource() instanceof DataExpression) { + DataExpression dexpr = (DataExpression) ((AssignmentStatement) s).getSource(); + if(dexpr.isRead()) { String pfname = dexpr.getVarParam(DataExpression.IO_FILENAME).toString(); // found read-after-write - if (pWrites.containsKey(pfname) && !pfname.trim().isEmpty()) { + if(pWrites.containsKey(pfname) && !pfname.trim().isEmpty()) { // update read with essential write meta data DataIdentifier di = pWrites.get(pfname); FileFormat ft = (di.getFileFormat() != null) ? di.getFileFormat() : FileFormat.TEXT; dexpr.addVarParam(DataExpression.FORMAT_TYPE, new StringIdentifier(ft.toString(), di)); - if (di.getDim1() >= 0) + if(di.getDim1() >= 0) dexpr.addVarParam(DataExpression.READROWPARAM, new IntIdentifier(di.getDim1(), di)); - if (di.getDim2() >= 0) + if(di.getDim2() >= 0) dexpr.addVarParam(DataExpression.READCOLPARAM, new IntIdentifier(di.getDim2(), di)); - if (di.getValueType() != ValueType.UNKNOWN) + if(di.getValueType() != ValueType.UNKNOWN) dexpr.addVarParam(DataExpression.VALUETYPEPARAM, - new StringIdentifier(di.getValueType().toExternalString(), di)); - if (di.getDataType() != DataType.UNKNOWN) + new StringIdentifier(di.getValueType().toExternalString(), di)); + if(di.getDataType() != DataType.UNKNOWN) dexpr.addVarParam(DataExpression.DATATYPEPARAM, - new StringIdentifier(di.getDataType().toString(), di)); + new StringIdentifier(di.getDataType().toString(), di)); ret = true; } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/InstructionUtils.java b/src/main/java/org/apache/sysds/runtime/instructions/InstructionUtils.java index da3de02419d..c863e953b1f 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/InstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/InstructionUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -98,68 +98,71 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator; import org.apache.sysds.runtime.matrix.operators.UnarySketchOperator; - public class InstructionUtils { protected static final Log LOG = LogFactory.getLog(InstructionUtils.class.getName()); //thread-local string builders for instruction concatenation (avoid allocation) private static ThreadLocal _strBuilders = new ThreadLocal<>() { @Override - protected StringBuilder initialValue() { + protected StringBuilder initialValue() { return new StringBuilder(64); } }; - + public static StringBuilder getStringBuilder() { StringBuilder sb = _strBuilders.get(); sb.setLength(0); //reuse allocated space return sb; } - - public static int checkNumFields( String str, int expected ) { + + public static int checkNumFields(String str, int expected) { //note: split required for empty tokens int numParts = str.split(Instruction.OPERAND_DELIM).length; int numFields = numParts - 2; // -2 accounts for execType and opcode - - if ( numFields != expected ) - throw new DMLRuntimeException("checkNumFields() for (" + str + ") -- expected number (" + expected + ") != is not equal to actual number (" + numFields + ")."); - - return numFields; + + if(numFields != expected) + throw new DMLRuntimeException("checkNumFields() for (" + str + ") -- expected number (" + expected + + ") != is not equal to actual number (" + numFields + ")."); + + return numFields; } - public static int checkNumFields( String[] parts, int expected ) { + public static int checkNumFields(String[] parts, int expected) { int numParts = parts.length; int numFields = numParts - 1; //account for opcode - - if ( numFields != expected ) - throw new DMLRuntimeException("checkNumFields() -- expected number (" + expected + ") != is not equal to actual number (" + numFields + ")."); - - return numFields; + + if(numFields != expected) + throw new DMLRuntimeException( + "checkNumFields() -- expected number (" + expected + ") != is not equal to actual number (" + + numFields + ")."); + + return numFields; } - public static int checkNumFields( String[] parts, int expected1, int expected2 ) { + public static int checkNumFields(String[] parts, int expected1, int expected2) { int numParts = parts.length; int numFields = numParts - 1; //account for opcode - - if ( numFields != expected1 && numFields != expected2 ) - throw new DMLRuntimeException("checkNumFields() -- expected number (" + expected1 + " or "+ expected2 +") != is not equal to actual number (" + numFields + ")."); - - return numFields; + + if(numFields != expected1 && numFields != expected2) + throw new DMLRuntimeException("checkNumFields() -- expected number (" + expected1 + " or " + expected2 + + ") != is not equal to actual number (" + numFields + ")."); + + return numFields; } - public static int checkNumFields( String[] parts, int... expected ) { + public static int checkNumFields(String[] parts, int... expected) { int numParts = parts.length; int numFields = numParts - 1; //account for opcode return checkMatchingNumField(numFields, expected); } - private static int checkMatchingNumField(int numFields, int... expected){ - if (Arrays.stream(expected).noneMatch((i) -> numFields == i)) { + private static int checkMatchingNumField(int numFields, int... expected) { + if(Arrays.stream(expected).noneMatch((i) -> numFields == i)) { StringBuilder sb = new StringBuilder(); sb.append("checkNumFields() -- expected number ("); - for (int i = 0; i < expected.length; i++) { + for(int i = 0; i < expected.length; i++) { sb.append(expected[i]); - if (i != expected.length - 1) + if(i != expected.length - 1) sb.append(", "); } sb.append(") != is not equal to actual number (").append(numFields).append(")."); @@ -168,88 +171,87 @@ private static int checkMatchingNumField(int numFields, int... expected){ return numFields; } - public static int checkNumFields( String str, int... expected ) { + public static int checkNumFields(String str, int... expected) { int numParts = str.split(Instruction.OPERAND_DELIM).length; int numFields = numParts - 2; // -2 accounts for execType and opcode return checkMatchingNumField(numFields, expected); } - public static int checkNumFields( String str, int expected1, int expected2 ) { + public static int checkNumFields(String str, int expected1, int expected2) { //note: split required for empty tokens int numParts = str.split(Instruction.OPERAND_DELIM).length; int numFields = numParts - 2; // -2 accounts for execType and opcode - if ( numFields != expected1 && numFields != expected2 ) - throw new DMLRuntimeException("checkNumFields() for (" + str + ") -- expected number (" + expected1 + " or "+ expected2 +") != is not equal to actual number (" + numFields + ")."); - return numFields; + if(numFields != expected1 && numFields != expected2) + throw new DMLRuntimeException( + "checkNumFields() for (" + str + ") -- expected number (" + expected1 + " or " + expected2 + + ") != is not equal to actual number (" + numFields + ")."); + return numFields; } - + /** - * Given an instruction string, strip-off the execution type and return - * opcode and all input/output operands WITHOUT their data/value type. - * i.e., ret.length = parts.length-1 (-1 for execution type) - * + * Given an instruction string, strip-off the execution type and return opcode and all input/output operands WITHOUT + * their data/value type. i.e., ret.length = parts.length-1 (-1 for execution type) + * * @param str instruction string * @return instruction parts as string array */ - public static String[] getInstructionParts( String str ) { - StringTokenizer st = new StringTokenizer( str, Instruction.OPERAND_DELIM ); - String[] ret = new String[st.countTokens()-1]; + public static String[] getInstructionParts(String str) { + StringTokenizer st = new StringTokenizer(str, Instruction.OPERAND_DELIM); + String[] ret = new String[st.countTokens() - 1]; st.nextToken(); // stripping-off the exectype ret[0] = st.nextToken(); // opcode int index = 1; - while( st.hasMoreTokens() ){ + while(st.hasMoreTokens()) { String tmp = st.nextToken(); int ix = tmp.indexOf(Instruction.DATATYPE_PREFIX); - ret[index++] = tmp.substring(0,((ix>=0)?ix:tmp.length())); + ret[index++] = tmp.substring(0, ((ix >= 0) ? ix : tmp.length())); } return ret; } - + /** - * Given an instruction string, this function strips-off the - * execution type (CP or SPARK) and returns the remaining parts, - * which include the opcode as well as the input and output operands. - * Each returned part will have the datatype and valuetype associated - * with the operand. - * + * Given an instruction string, this function strips-off the execution type (CP or SPARK) and returns the remaining + * parts, which include the opcode as well as the input and output operands. Each returned part will have the + * datatype and valuetype associated with the operand. + * * This function is invoked mainly for parsing CPInstructions. - * + * * @param str instruction string * @return instruction parts as string array */ - public static String[] getInstructionPartsWithValueType( String str ) { + public static String[] getInstructionPartsWithValueType(String str) { //note: split required for empty tokens String[] parts = str.split(Instruction.OPERAND_DELIM, -1); - String[] ret = new String[parts.length-1]; // stripping-off the exectype + String[] ret = new String[parts.length - 1]; // stripping-off the exectype ret[0] = parts[1]; // opcode - for( int i=1; i= parts.length ) - throw new DMLRuntimeException("Operand position " - + operand + " exceeds the length of the instruction."); + if(operand >= parts.length) + throw new DMLRuntimeException("Operand position " + operand + " exceeds the length of the instruction."); //replace and reconstruct string parts[operand] = newValue; return concatOperands(parts); @@ -1128,27 +1178,26 @@ public static String replaceOperand(String instStr, int operand, String newValue public static String removeOperand(String instStr, int operand) { //split instruction and check for correctness String[] parts = instStr.split(Lop.OPERAND_DELIMITOR); - if( operand >= parts.length ) - throw new DMLRuntimeException("Operand position " - + operand + " exceeds the length of the instruction."); + if(operand >= parts.length) + throw new DMLRuntimeException("Operand position " + operand + " exceeds the length of the instruction."); //remove and reconstruct string return concatOperands(ArrayUtils.remove(parts, operand)); } public static String replaceOperandName(String instStr) { String[] parts = instStr.split(Lop.OPERAND_DELIMITOR); - String oldName = parts[parts.length-1]; + String oldName = parts[parts.length - 1]; String[] Nameparts = oldName.split(Instruction.VALUETYPE_PREFIX); Nameparts[0] = "xxx"; String newName = concatOperandParts(Nameparts); - parts[parts.length-1] = newName; + parts[parts.length - 1] = newName; return concatOperands(parts); } /** - * Concat the inputs as operands to generate the instruction string. - * The inputs are separated by the operand delimiter and appended - * using a ThreadLocal StringBuilder. + * Concat the inputs as operands to generate the instruction string. The inputs are separated by the operand + * delimiter and appended using a ThreadLocal StringBuilder. + * * @param inputs operand inputs given as strings * @return the instruction string with the given inputs concatenated */ @@ -1157,13 +1206,14 @@ public static String concatOperands(String... inputs) { sb.setLength(0); //reuse allocated space return concatOperands(sb, inputs); } - + public static String concatOperands(StringBuilder sb, String... inputs) { return concatBaseOperandsWithDelim(sb, Lop.OPERAND_DELIMITOR, inputs); } /** * Concat the input parts with the value type delimiter. + * * @param inputs input operand parts as strings * @return concatenated input parts */ @@ -1173,30 +1223,33 @@ public static String concatOperandParts(String... inputs) { return concatBaseOperandsWithDelim(sb, Instruction.VALUETYPE_PREFIX, inputs); } - private static String concatBaseOperandsWithDelim(StringBuilder sb, String delim, String... inputs){ - for( int i=0; i fb = sec.getSparkContext().broadcast(sec.getFrameInput(input2.getName())); + out = in1.mapValues(new setColumnNames(fb.getValue())); + sec.releaseFrameInput(input2.getName()); + } else { JavaPairRDD in2 = sec.getFrameBinaryBlockRDDHandleForVariable(input2.getName()); // create output frame @@ -140,4 +145,22 @@ public FrameBlock call(FrameBlock arg0) throws Exception { return arg0.applySchema(schema); } } + + private static class setColumnNames implements Function{ + //private static final long serialVersionUID = 1L; + + private String[] columnNames; + + public setColumnNames(FrameBlock names) { + columnNames = new String[names.getNumColumns()]; + for(int i = 0; i < columnNames.length; i++) + columnNames[i] = names.get(0, i).toString(); + } + + @Override + public FrameBlock call(FrameBlock arg0) throws Exception { + arg0.setColumnNames(columnNames); + return arg0; + } + } } \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/frame/FrameColumnNamesTest.java b/src/test/java/org/apache/sysds/test/functions/frame/FrameColumnNamesTest.java index d1ee4215e1a..e72ba519d81 100644 --- a/src/test/java/org/apache/sysds/test/functions/frame/FrameColumnNamesTest.java +++ b/src/test/java/org/apache/sysds/test/functions/frame/FrameColumnNamesTest.java @@ -44,6 +44,8 @@ @net.jcip.annotations.NotThreadSafe public class FrameColumnNamesTest extends AutomatedTestBase { private final static String TEST_NAME = "ColumnNames"; + private final static String TEST_NAME_GET = "GetNames"; + private final static String TEST_NAME_SET = "SetNames"; private final static String TEST_DIR = "functions/frame/"; private static final String TEST_CLASS_DIR = TEST_DIR + FrameColumnNamesTest.class.getSimpleName() + "/"; @@ -60,6 +62,9 @@ public static Collection data() { @Override public void setUp() { addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"B"})); + addTestConfiguration(TEST_NAME_GET, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME_GET, new String[] {"B"})); + addTestConfiguration(TEST_NAME_SET, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME_SET, new String[] {"B"})); + } @Test @@ -72,6 +77,134 @@ public void testDetectSchemaDoubleSpark() { runGetColNamesTest(_columnNames, ExecType.SPARK); } + @Test + public void testGetNamesCP() { + runGetNamesTest(_columnNames, ExecType.CP); + } + + @Test + public void testGetNamesSpark() { + runGetNamesTest(_columnNames, ExecType.SPARK); + } + + @Test + public void testSetNamesCP() { + runSetNamesTest(_columnNames, ExecType.CP); + } + + @Test + public void testSetNamesSpark() { + runSetNamesTest(_columnNames, ExecType.SPARK); + } + + private void runGetNamesTest(String[] columnNames, ExecType et) { + Types.ExecMode platformOld = setExecMode(et); + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + + if(et == ExecType.SPARK) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + setOutputBuffering(true); + try { + getAndLoadTestConfiguration(TEST_NAME_GET); + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME_GET + ".dml"; + programArgs = new String[] {"-args", input("A"), String.valueOf(_rows), + Integer.toString(columnNames.length), output("B")}; + + Types.ValueType[] schema = Collections.nCopies( + columnNames.length, Types.ValueType.FP64).toArray(new Types.ValueType[0]); + FrameBlock frame1 = new FrameBlock(schema); + frame1.setColumnNames(columnNames); + FrameWriter writer = FrameWriterFactory.createFrameWriter(FileFormat.CSV, + new FileFormatPropertiesCSV(true, ",", false)); + + double[][] A = getRandomMatrix(_rows, schema.length, Double.MIN_VALUE, Double.MAX_VALUE, 0.7, 14123); + TestUtils.initFrameData(frame1, A, schema, _rows); + writer.writeFrameToHDFS(frame1, input("A"), _rows, schema.length); + + runTest(true, false, null, -1); + + FrameBlock resultFrame = + readDMLFrameFromHDFS("B", FileFormat.BINARY); + + Assert.assertEquals( + "Unexpected number of result rows.", + 1, + resultFrame.getNumRows()); + + Assert.assertEquals( + "Unexpected number of result columns.", + columnNames.length, + resultFrame.getNumColumns()); + + // verify output schema + for(int i = 0; i < schema.length; i++) { + Assert + .assertEquals("Wrong result: " + columnNames[i] + ".", columnNames[i], resultFrame.get(0, i).toString()); + } + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + finally { + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + } + } + + private void runSetNamesTest(String[] columnNames, ExecType et) { + Types.ExecMode platformOld = setExecMode(et); + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + + if(et == ExecType.SPARK) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + setOutputBuffering(true); + try { + getAndLoadTestConfiguration(TEST_NAME_SET); + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME_SET + ".dml"; + programArgs = new String[] {"-args",input("X"),String.valueOf(_rows),Integer.toString(columnNames.length), + input("N"),output("B") + }; + + Types.ValueType[] schema = Collections.nCopies( + columnNames.length, Types.ValueType.FP64).toArray(new Types.ValueType[0]); + + FrameBlock frame1 = new FrameBlock(schema); + FrameWriter writer = FrameWriterFactory.createFrameWriter(FileFormat.CSV, + new FileFormatPropertiesCSV(true, ",", false)); + + double[][] A = getRandomMatrix(_rows, schema.length, Double.MIN_VALUE, Double.MAX_VALUE, 0.7, 14123); + TestUtils.initFrameData(frame1, A, schema, _rows); + writer.writeFrameToHDFS(frame1, input("X"), _rows, schema.length); + + Types.ValueType[] nameSchema = Collections.nCopies( + columnNames.length, Types.ValueType.STRING).toArray(new Types.ValueType[0]); + + FrameBlock names = new FrameBlock(nameSchema); + names.ensureAllocatedColumns(1); + for(int i = 0; i < columnNames.length; i++) + names.set(0, i, columnNames[i]); + FrameWriter nameWriter = FrameWriterFactory.createFrameWriter(FileFormat.CSV, + new FileFormatPropertiesCSV(false, ",", false)); + nameWriter.writeFrameToHDFS(names, input("N"), 1, columnNames.length); + + runTest(true, false, null, -1); + + FrameBlock frame2 = readDMLFrameFromHDFS("B", FileFormat.BINARY); + for(int i = 0; i < columnNames.length; i++) + Assert.assertEquals("Wrong result: " + columnNames[i] + ".", columnNames[i], frame2.get(0, i).toString()); + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + finally { + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + } + } + + private void runGetColNamesTest(String[] columnNames, ExecType et) { Types.ExecMode platformOld = setExecMode(et); boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; diff --git a/src/test/scripts/functions/frame/ColNameCbindPropagation.dml b/src/test/scripts/functions/frame/ColNameCbindPropagation.dml new file mode 100644 index 00000000000..e46a4a7dbe8 --- /dev/null +++ b/src/test/scripts/functions/frame/ColNameCbindPropagation.dml @@ -0,0 +1,26 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X1 = read($1, rows=$2, cols=$3, data_type="frame", format="csv", header=TRUE); +X2 = read($4, rows=$2, cols=$5, data_type="frame", format="csv", header=TRUE); +Y = cbind(X1, X2); +B = getNames(Y); +write(B, $6, format="binary"); \ No newline at end of file diff --git a/src/test/scripts/functions/frame/ColNameRbindPropagation.dml b/src/test/scripts/functions/frame/ColNameRbindPropagation.dml new file mode 100644 index 00000000000..e11892f3645 --- /dev/null +++ b/src/test/scripts/functions/frame/ColNameRbindPropagation.dml @@ -0,0 +1,26 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X1 = read($1, rows=$2, cols=$3, data_type="frame", format="csv", header=TRUE); +X2 = read($4, rows=$2, cols=$5, data_type="frame", format="csv", header=TRUE); +Y = rbind(X1, X2); +B = getNames(Y); +write(B, $6, format="binary"); \ No newline at end of file diff --git a/src/test/scripts/functions/frame/ColNameSlicePropagation.dml b/src/test/scripts/functions/frame/ColNameSlicePropagation.dml new file mode 100644 index 00000000000..647e4f172d5 --- /dev/null +++ b/src/test/scripts/functions/frame/ColNameSlicePropagation.dml @@ -0,0 +1,25 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1, rows=$2, cols=$3, data_type="frame", format="csv", header=TRUE); +Y = X[,2:($3-1)]; +B = getNames(Y); +write(B, $4, format="binary"); \ No newline at end of file diff --git a/src/test/scripts/functions/frame/GetNames.dml b/src/test/scripts/functions/frame/GetNames.dml new file mode 100644 index 00000000000..70b8f22d8d9 --- /dev/null +++ b/src/test/scripts/functions/frame/GetNames.dml @@ -0,0 +1,24 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1, rows=$2, cols=$3, data_type="frame", format="csv", header=TRUE); +R = getNames(X); +write(R, $4, format="binary"); \ No newline at end of file diff --git a/src/test/scripts/functions/frame/SetNames.dml b/src/test/scripts/functions/frame/SetNames.dml new file mode 100644 index 00000000000..157a415babc --- /dev/null +++ b/src/test/scripts/functions/frame/SetNames.dml @@ -0,0 +1,28 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1, rows=$2, cols=$3, data_type="frame", format="csv", header=TRUE); +N = read($4, rows=1, cols=$3, data_type="frame", format="csv", header=FALSE); + +X2 = setNames(X, N) +B = getNames(X2) + +write(B, $5, format="binary"); \ No newline at end of file