diff --git a/mathics/builtin/files_io/files.py b/mathics/builtin/files_io/files.py index 7245d9242..0c12a0849 100644 --- a/mathics/builtin/files_io/files.py +++ b/mathics/builtin/files_io/files.py @@ -40,7 +40,14 @@ ) from mathics.eval.directories import TMP_DIR from mathics.eval.encoding import CHARACTER_ENCODING_MAP -from mathics.eval.files_io.files import eval_Close, eval_Get, eval_Open, eval_Read +from mathics.eval.files_io.files import ( + eval_Close, + eval_DumpParse, + eval_Get, + eval_Get_from_PCL, + eval_Open, + eval_Read, +) from mathics.eval.files_io.read import ( Mathics3Open, channel_to_stream, @@ -220,6 +227,96 @@ def eval(self, obj, evaluation: Evaluation): return eval_Close(obj, evaluation) +class DumpParse(Builtin): + """ + ## :trace native symbol: + +
+
'DumpParse'[$input$, $output$, $Options$] +
Reads Mathics3 source text $input$, parses it and write the the M-expressions \ + a Pickle file $output$. $True$ is returned if everthing want okay. +
+ + S> dumpParsedFile = FileNameJoin[{$TemporaryDirectory, "BoolEval.mx3"}] + = ... + S> DumpParse["BoolEval/BoolEval.m", dumpParsedFile] + = True + #> Clear[dumpParsedFile] + """ + + options = { + "CharacterEncoding": "Null", + "Path": "Null", + "Trace": "False", + } + summary_text = ( + "Read and parse Mathics3 source text, and then write the parse to a PCL file" + ) + + def eval( + self, input: String, output: String, evaluation: Evaluation, options: dict + ): + "DumpParse[input_String, output_String, OptionsPattern[DumpParse]]" + + # Make sure to pick up copy from module each time instead of using + # use "from ... import DEFAULT_TRACE_FN" which will not pick + # up run-time changes made to the module function. + trace_fn = io_files.DEFAULT_TRACE_FN + + trace_get = evaluation.parse("Settings`$TraceGet") + if ( + options["System`Trace"].to_python() + or trace_get.evaluate(evaluation) is SymbolTrue + ): + trace_fn = io_files.GET_PRINT_FN + # Process the "Path" option. + # The result will be put in py_path_directories + path_directories = options["System`Path"] + py_path_directories = None + if ( + path_directories is not SymbolNull + and (py_path_directories := path_directories.to_python(string_quotes=False)) + is not None + ): + if isinstance(py_path_directories, tuple): + for dir in py_path_directories: + if not isinstance(dir, str): + evaluation.message("DumpParse", "path", dir) + py_path_directories = None + break + elif isinstance(py_path_directories, str): + py_path_directories = [py_path_directories] + else: + evaluation.message("DumpParse", "path", path_directories) + py_path_directories = None + + # Process the "CharacterEncoding" option. + encoding = options["System`CharacterEncoding"] + py_current_encoding = evaluation.definitions.get_ownvalue( + "System`$CharacterEncoding" + ).value + if isinstance(encoding, String): + py_encoding = encoding.to_python(string_quotes=False) + if py_encoding not in CHARACTER_ENCODING_MAP: + # "noopen" matches WMA. This is nonsensical. + evaluation.message("Get", "noopen", encoding) + py_encoding = py_current_encoding + else: + if encoding is not SymbolNull: + evaluation.message("$CharacterEncoding", "charcode", encoding) + py_encoding = py_current_encoding + + # Dump perform the actual evaluation + return eval_DumpParse( + input.value, + output.value, + evaluation, + py_encoding, + trace_fn, + py_path_directories, + ) + + class EndOfFile(Builtin): """ :WMA link: @@ -418,7 +515,7 @@ def eval(self, path: String, evaluation: Evaluation, options: dict): if isinstance(py_path_directories, tuple): for dir in py_path_directories: if not isinstance(dir, str): - evaluation.message("Put", "path", dir) + evaluation.message("Get", "path", dir) py_path_directories = None break elif isinstance(py_path_directories, str): @@ -444,6 +541,15 @@ def eval(self, path: String, evaluation: Evaluation, options: dict): py_encoding = py_current_encoding # perform the actual evaluation + if path.value.endswith("mx3"): + return eval_Get_from_PCL( + path.value, + evaluation, + py_encoding, + trace_fn, + py_path_directories, + ) + return eval_Get( path.value, evaluation, diff --git a/mathics/core/parser/util.py b/mathics/core/parser/util.py index ece35aa21..770641677 100644 --- a/mathics/core/parser/util.py +++ b/mathics/core/parser/util.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import pickle from typing import FrozenSet, Optional, Tuple import mathics_scanner.location @@ -8,14 +9,31 @@ from mathics.core.definitions import Definitions from mathics.core.element import BaseElement +from mathics.core.parser.ast import Node as ASTNode from mathics.core.parser.convert import convert from mathics.core.parser.feed import MathicsSingleLineFeeder from mathics.core.parser.parser import Parser -from mathics.core.symbols import Symbol, ensure_context +from mathics.core.symbols import Symbol, SymbolTrue, ensure_context +from mathics.core.systemsymbols import SymbolFailed parser = Parser() +def dump_exprs_to_pcl_file(exprs, pickle_file: str) -> Symbol: + """ + Parse input from `feeder` and pickle serialize the parsed M-expression Python written + to pickle_file. + Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle. + + Return SymbolTrue if things went okay. + """ + # Open the file in binary write mode + with open(pickle_file, "wb") as f: + # Protocol -1 uses the highest available binary protocol for efficiency + pickle.dump(exprs, f, protocol=pickle.HIGHEST_PROTOCOL) + return SymbolTrue + + def parse(definitions, feeder: LineFeeder) -> Optional[BaseElement]: """ Parse input (from the frontend, -e, input files, ToExpression etc). @@ -85,6 +103,36 @@ def parse_returning_code( return converted, source_text +def parse_dump_to_pcl_file(feeder: LineFeeder, pickle_file: str) -> Symbol: + """ + Parse input from `feeder` and pickle serialize the parsed M-expression Python written + to pickle_file. + Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle. + """ + ast = parser.parse(feeder) + + if ast is None: + return SymbolFailed + # Ensure the input is actually a Node (optional safety check) + if not isinstance(ast, ASTNode): + raise TypeError(f"Expected mathics.core.parser.ast.Node, got {type(ast)}") + + # Open the file in binary write mode + with open(pickle_file, "wb") as f: + # Protocol -1 uses the highest available binary protocol for efficiency + pickle.dump(ast, f, protocol=pickle.HIGHEST_PROTOCOL) + return SymbolTrue + + +def parse_from_pcl_file(definitions, pickle_file: str): + + with open(pickle_file, "rb") as f: + # Load the object from the binary file. + result = pickle.load(f) + + return result + + class SystemDefinitions: """ Dummy Definitions object that puts every unqualified symbol in diff --git a/mathics/eval/files_io/files.py b/mathics/eval/files_io/files.py index a417ec66c..5560d95a9 100644 --- a/mathics/eval/files_io/files.py +++ b/mathics/eval/files_io/files.py @@ -23,9 +23,13 @@ from mathics.core.evaluation import Evaluation from mathics.core.expression import BaseElement, Expression from mathics.core.parser import MathicsFileLineFeeder, MathicsMultiLineFeeder -from mathics.core.parser.util import parse_incrementally_by_line +from mathics.core.parser.util import ( + dump_exprs_to_pcl_file, + parse_from_pcl_file, + parse_incrementally_by_line, +) from mathics.core.streams import path_search, stream_manager -from mathics.core.symbols import Symbol, SymbolNull +from mathics.core.symbols import Symbol, SymbolNull, SymbolTrue from mathics.core.systemsymbols import ( SymbolEndOfFile, SymbolExpression, @@ -117,6 +121,84 @@ def eval_Close(obj, evaluation: Evaluation): return name +def eval_DumpParse( + input_path: str, + output_path: str, + evaluation: Evaluation, + encoding: str, + trace_fn: Optional[Callable] = DEFAULT_TRACE_FN, + path_directories: Optional[Sequence[str]] = None, +) -> Symbol: + """ + Reads file `input_path`, parses each expression in the file + accumulating them in a list. This list is Python Pickled and + written to `output_path`. True is returned if everything succeeded. + """ + + if path_directories is None: + path_directories = tuple(streams.PATH_VAR) + resolved_path, _ = path_search(input_path, path_directories) + if resolved_path is None: + resolved_path = input_path + definitions = evaluation.definitions + + # Wrap actual evaluation to handle setting $Input + # and $InputFileName + # store input paths of calling context + + global INPUT_VAR + outer_input_var = INPUT_VAR + outer_inputfile = definitions.get_inputfile() + + # Set a new input path. + INPUT_VAR = resolved_path + definitions.set_inputfile(INPUT_VAR) + + # Save old PATH_VAR in case it gets changed in running Get? + # This seems to be needed, but not 100% sure there isn't + # a better and more robust way. + old_streams_path_var = streams.PATH_VAR + streams.PATH_VAR = SymbolPath.evaluate(evaluation).to_python(string_quotes=False) + + queries = [] + + if trace_fn is not None: + trace_fn(0, resolved_path + "\n") + try: + with Mathics3Open(resolved_path, "r", encoding=encoding) as f: + feeder = MathicsFileLineFeeder(f, trace_fn) + while not feeder.empty(): + try: + # Note: we use mathics.core.parser.parse + # so that tracing/debugging can intercept parse() + query = mathics.core.parser.parse(definitions, feeder) + except SyntaxError: + return SymbolNull + finally: + feeder.send_messages(evaluation) + if query is None: # blank line / comment + continue + else: + queries.append(query) + + # result = query.evaluate(evaluation) + except IOError: + evaluation.message("DumpParse", "noopen", input_path) + return SymbolFailed + except MessageException as e: + e.message(evaluation) + return SymbolFailed + finally: + # Whether we had an exception or not, restore the input path + # and the state of definitions prior to calling Get. + INPUT_VAR = outer_input_var + definitions.set_inputfile(outer_inputfile) + streams.PATH_VAR = old_streams_path_var + + dump_exprs_to_pcl_file(queries, output_path) + return SymbolTrue + + def eval_Get( path: str, evaluation: Evaluation, @@ -186,6 +268,32 @@ def eval_Get( return result +def eval_Get_from_PCL( + path: str, + evaluation: Evaluation, + encoding: str, + trace_fn: Optional[Callable] = DEFAULT_TRACE_FN, + path_directories: Optional[Sequence[str]] = None, +): + """ + Reads a file and evaluates each expression, returning only the last one. + """ + + result = None + if path_directories is None: + path_directories = tuple(streams.PATH_VAR) + resolved_path, _ = path_search(path, path_directories) + if resolved_path is None: + resolved_path = path + + parse_list = parse_from_pcl_file(evaluation.definitions, resolved_path) + if not isinstance(parse_list, list): + return None + for query in parse_list: + result = query.evaluate(evaluation) + return result + + def eval_Open( name: String, mode: str,