From 091d172d33e3ade2e80e6c79ab0bc34dbba03a31 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 24 Apr 2026 22:37:25 -0400 Subject: [PATCH 1/3] Add parsed files and read them... Add DumpParse to read Mathics3 source text and write it Python pickled to a file. Extend Get so that it will unpickle a file. --- mathics/builtin/files_io/files.py | 107 +++++++++++++++++++++++++++- mathics/core/parser/util.py | 43 ++++++++++++ mathics/eval/files_io/files.py | 112 +++++++++++++++++++++++++++++- 3 files changed, 258 insertions(+), 4 deletions(-) diff --git a/mathics/builtin/files_io/files.py b/mathics/builtin/files_io/files.py index 7245d9242..c705362ae 100644 --- a/mathics/builtin/files_io/files.py +++ b/mathics/builtin/files_io/files.py @@ -40,7 +40,14 @@ ) from mathics.eval.directories import TMP_DIR from mathics.eval.encoding import CHARACTER_ENCODING_MAP -from mathics.eval.files_io.files import eval_Close, eval_Get, eval_Open, eval_Read +from mathics.eval.files_io.files import ( + eval_Close, + eval_DumpParse, + eval_Get, + eval_Get_from_PCL, + eval_Open, + eval_Read, +) from mathics.eval.files_io.read import ( Mathics3Open, channel_to_stream, @@ -220,6 +227,93 @@ def eval(self, obj, evaluation: Evaluation): return eval_Close(obj, evaluation) +class DumpParse(Builtin): + """ + ## :trace native symbol: + +
+
'DumpParse'[$input$, $output$, $Options$] +
Reads Mathics3 source text $input$, parses it and write the the M-expressions \ + a Pickle file $output$. $True$ is returned if everthing want okay. +
+ + >> DumpParse["BoolEval/BoolEval.m", "/tmp/BoolEval.mx3"] + = ... + """ + + options = { + "CharacterEncoding": "Null", + "Path": "Null", + "Trace": "False", + } + summary_text = ( + "Read and parse Mathics3 source text, and then write the parse to a PCL file" + ) + + def eval( + self, input: String, output: String, evaluation: Evaluation, options: dict + ): + "DumpParse[input_String, output_String, OptionsPattern[DumpParse]]" + + # Make sure to pick up copy from module each time instead of using + # use "from ... import DEFAULT_TRACE_FN" which will not pick + # up run-time changes made to the module function. + trace_fn = io_files.DEFAULT_TRACE_FN + + trace_get = evaluation.parse("Settings`$TraceGet") + if ( + options["System`Trace"].to_python() + or trace_get.evaluate(evaluation) is SymbolTrue + ): + trace_fn = io_files.GET_PRINT_FN + # Process the "Path" option. + # The result will be put in py_path_directories + path_directories = options["System`Path"] + py_path_directories = None + if ( + path_directories is not SymbolNull + and (py_path_directories := path_directories.to_python(string_quotes=False)) + is not None + ): + if isinstance(py_path_directories, tuple): + for dir in py_path_directories: + if not isinstance(dir, str): + evaluation.message("DumpParse", "path", dir) + py_path_directories = None + break + elif isinstance(py_path_directories, str): + py_path_directories = [py_path_directories] + else: + evaluation.message("DumpParse", "path", path_directories) + py_path_directories = None + + # Process the "CharacterEncoding" option. + encoding = options["System`CharacterEncoding"] + py_current_encoding = evaluation.definitions.get_ownvalue( + "System`$CharacterEncoding" + ).value + if isinstance(encoding, String): + py_encoding = encoding.to_python(string_quotes=False) + if py_encoding not in CHARACTER_ENCODING_MAP: + # "noopen" matches WMA. This is nonsensical. + evaluation.message("Get", "noopen", encoding) + py_encoding = py_current_encoding + else: + if encoding is not SymbolNull: + evaluation.message("$CharacterEncoding", "charcode", encoding) + py_encoding = py_current_encoding + + # Dump perform the actual evaluation + return eval_DumpParse( + input.value, + output.value, + evaluation, + py_encoding, + trace_fn, + py_path_directories, + ) + + class EndOfFile(Builtin): """ :WMA link: @@ -418,7 +512,7 @@ def eval(self, path: String, evaluation: Evaluation, options: dict): if isinstance(py_path_directories, tuple): for dir in py_path_directories: if not isinstance(dir, str): - evaluation.message("Put", "path", dir) + evaluation.message("Get", "path", dir) py_path_directories = None break elif isinstance(py_path_directories, str): @@ -444,6 +538,15 @@ def eval(self, path: String, evaluation: Evaluation, options: dict): py_encoding = py_current_encoding # perform the actual evaluation + if path.value.endswith("mx3"): + return eval_Get_from_PCL( + path.value, + evaluation, + py_encoding, + trace_fn, + py_path_directories, + ) + return eval_Get( path.value, evaluation, diff --git a/mathics/core/parser/util.py b/mathics/core/parser/util.py index ece35aa21..6a00db77c 100644 --- a/mathics/core/parser/util.py +++ b/mathics/core/parser/util.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import pickle from typing import FrozenSet, Optional, Tuple import mathics_scanner.location @@ -8,6 +9,7 @@ from mathics.core.definitions import Definitions from mathics.core.element import BaseElement +from mathics.core.parser.ast import Node as ASTNode from mathics.core.parser.convert import convert from mathics.core.parser.feed import MathicsSingleLineFeeder from mathics.core.parser.parser import Parser @@ -16,6 +18,18 @@ parser = Parser() +def dump_exprs_to_pcl_file(exprs, pickle_file: str) -> Optional[str]: + """ + Parse input from `feeder` and pickle serialize the parsed M-expression Python written + to pickle_file. + Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle. + """ + # Open the file in binary write mode + with open(pickle_file, "wb") as f: + # Protocol -1 uses the highest available binary protocol for efficiency + pickle.dump(exprs, f, protocol=pickle.HIGHEST_PROTOCOL) + + def parse(definitions, feeder: LineFeeder) -> Optional[BaseElement]: """ Parse input (from the frontend, -e, input files, ToExpression etc). @@ -85,6 +99,35 @@ def parse_returning_code( return converted, source_text +def parse_dump_to_pcl_file(feeder: LineFeeder, pickle_file: str) -> Optional[str]: + """ + Parse input from `feeder` and pickle serialize the parsed M-expression Python written + to pickle_file. + Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle. + """ + ast = parser.parse(feeder) + + if ast is None: + return None + # Ensure the input is actually a Node (optional safety check) + if not isinstance(ast, ASTNode): + raise TypeError(f"Expected mathics.core.parser.ast.Node, got {type(ast)}") + + # Open the file in binary write mode + with open(pickle_file, "wb") as f: + # Protocol -1 uses the highest available binary protocol for efficiency + pickle.dump(ast, f, protocol=pickle.HIGHEST_PROTOCOL) + + +def parse_from_pcl_file(definitions, pickle_file: str): + + with open(pickle_file, "rb") as f: + # Load the object from the binary file. + result = pickle.load(f) + + return result + + class SystemDefinitions: """ Dummy Definitions object that puts every unqualified symbol in diff --git a/mathics/eval/files_io/files.py b/mathics/eval/files_io/files.py index a417ec66c..5560d95a9 100644 --- a/mathics/eval/files_io/files.py +++ b/mathics/eval/files_io/files.py @@ -23,9 +23,13 @@ from mathics.core.evaluation import Evaluation from mathics.core.expression import BaseElement, Expression from mathics.core.parser import MathicsFileLineFeeder, MathicsMultiLineFeeder -from mathics.core.parser.util import parse_incrementally_by_line +from mathics.core.parser.util import ( + dump_exprs_to_pcl_file, + parse_from_pcl_file, + parse_incrementally_by_line, +) from mathics.core.streams import path_search, stream_manager -from mathics.core.symbols import Symbol, SymbolNull +from mathics.core.symbols import Symbol, SymbolNull, SymbolTrue from mathics.core.systemsymbols import ( SymbolEndOfFile, SymbolExpression, @@ -117,6 +121,84 @@ def eval_Close(obj, evaluation: Evaluation): return name +def eval_DumpParse( + input_path: str, + output_path: str, + evaluation: Evaluation, + encoding: str, + trace_fn: Optional[Callable] = DEFAULT_TRACE_FN, + path_directories: Optional[Sequence[str]] = None, +) -> Symbol: + """ + Reads file `input_path`, parses each expression in the file + accumulating them in a list. This list is Python Pickled and + written to `output_path`. True is returned if everything succeeded. + """ + + if path_directories is None: + path_directories = tuple(streams.PATH_VAR) + resolved_path, _ = path_search(input_path, path_directories) + if resolved_path is None: + resolved_path = input_path + definitions = evaluation.definitions + + # Wrap actual evaluation to handle setting $Input + # and $InputFileName + # store input paths of calling context + + global INPUT_VAR + outer_input_var = INPUT_VAR + outer_inputfile = definitions.get_inputfile() + + # Set a new input path. + INPUT_VAR = resolved_path + definitions.set_inputfile(INPUT_VAR) + + # Save old PATH_VAR in case it gets changed in running Get? + # This seems to be needed, but not 100% sure there isn't + # a better and more robust way. + old_streams_path_var = streams.PATH_VAR + streams.PATH_VAR = SymbolPath.evaluate(evaluation).to_python(string_quotes=False) + + queries = [] + + if trace_fn is not None: + trace_fn(0, resolved_path + "\n") + try: + with Mathics3Open(resolved_path, "r", encoding=encoding) as f: + feeder = MathicsFileLineFeeder(f, trace_fn) + while not feeder.empty(): + try: + # Note: we use mathics.core.parser.parse + # so that tracing/debugging can intercept parse() + query = mathics.core.parser.parse(definitions, feeder) + except SyntaxError: + return SymbolNull + finally: + feeder.send_messages(evaluation) + if query is None: # blank line / comment + continue + else: + queries.append(query) + + # result = query.evaluate(evaluation) + except IOError: + evaluation.message("DumpParse", "noopen", input_path) + return SymbolFailed + except MessageException as e: + e.message(evaluation) + return SymbolFailed + finally: + # Whether we had an exception or not, restore the input path + # and the state of definitions prior to calling Get. + INPUT_VAR = outer_input_var + definitions.set_inputfile(outer_inputfile) + streams.PATH_VAR = old_streams_path_var + + dump_exprs_to_pcl_file(queries, output_path) + return SymbolTrue + + def eval_Get( path: str, evaluation: Evaluation, @@ -186,6 +268,32 @@ def eval_Get( return result +def eval_Get_from_PCL( + path: str, + evaluation: Evaluation, + encoding: str, + trace_fn: Optional[Callable] = DEFAULT_TRACE_FN, + path_directories: Optional[Sequence[str]] = None, +): + """ + Reads a file and evaluates each expression, returning only the last one. + """ + + result = None + if path_directories is None: + path_directories = tuple(streams.PATH_VAR) + resolved_path, _ = path_search(path, path_directories) + if resolved_path is None: + resolved_path = path + + parse_list = parse_from_pcl_file(evaluation.definitions, resolved_path) + if not isinstance(parse_list, list): + return None + for query in parse_list: + result = query.evaluate(evaluation) + return result + + def eval_Open( name: String, mode: str, From 60a519db25e05382f6db943103217f46c3f84751 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 25 Apr 2026 06:49:51 -0400 Subject: [PATCH 2/3] Sandbox DumpParse tests... and add a return value to DumpParse --- mathics/builtin/files_io/files.py | 6 ++++-- mathics/core/parser/util.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mathics/builtin/files_io/files.py b/mathics/builtin/files_io/files.py index c705362ae..642a1df07 100644 --- a/mathics/builtin/files_io/files.py +++ b/mathics/builtin/files_io/files.py @@ -237,8 +237,10 @@ class DumpParse(Builtin): a Pickle file $output$. $True$ is returned if everthing want okay. - >> DumpParse["BoolEval/BoolEval.m", "/tmp/BoolEval.mx3"] - = ... + >S dumpParsedFile = FileNameJoin[{$TemporaryDirectory, "BoolEval.mx3"}] + >S DumpParse["BoolEval/BoolEval.m", dumpParsedFile] + = True + #> Clear[dumpParsedFile] """ options = { diff --git a/mathics/core/parser/util.py b/mathics/core/parser/util.py index 6a00db77c..770641677 100644 --- a/mathics/core/parser/util.py +++ b/mathics/core/parser/util.py @@ -13,21 +13,25 @@ from mathics.core.parser.convert import convert from mathics.core.parser.feed import MathicsSingleLineFeeder from mathics.core.parser.parser import Parser -from mathics.core.symbols import Symbol, ensure_context +from mathics.core.symbols import Symbol, SymbolTrue, ensure_context +from mathics.core.systemsymbols import SymbolFailed parser = Parser() -def dump_exprs_to_pcl_file(exprs, pickle_file: str) -> Optional[str]: +def dump_exprs_to_pcl_file(exprs, pickle_file: str) -> Symbol: """ Parse input from `feeder` and pickle serialize the parsed M-expression Python written to pickle_file. Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle. + + Return SymbolTrue if things went okay. """ # Open the file in binary write mode with open(pickle_file, "wb") as f: # Protocol -1 uses the highest available binary protocol for efficiency pickle.dump(exprs, f, protocol=pickle.HIGHEST_PROTOCOL) + return SymbolTrue def parse(definitions, feeder: LineFeeder) -> Optional[BaseElement]: @@ -99,7 +103,7 @@ def parse_returning_code( return converted, source_text -def parse_dump_to_pcl_file(feeder: LineFeeder, pickle_file: str) -> Optional[str]: +def parse_dump_to_pcl_file(feeder: LineFeeder, pickle_file: str) -> Symbol: """ Parse input from `feeder` and pickle serialize the parsed M-expression Python written to pickle_file. @@ -108,7 +112,7 @@ def parse_dump_to_pcl_file(feeder: LineFeeder, pickle_file: str) -> Optional[str ast = parser.parse(feeder) if ast is None: - return None + return SymbolFailed # Ensure the input is actually a Node (optional safety check) if not isinstance(ast, ASTNode): raise TypeError(f"Expected mathics.core.parser.ast.Node, got {type(ast)}") @@ -117,6 +121,7 @@ def parse_dump_to_pcl_file(feeder: LineFeeder, pickle_file: str) -> Optional[str with open(pickle_file, "wb") as f: # Protocol -1 uses the highest available binary protocol for efficiency pickle.dump(ast, f, protocol=pickle.HIGHEST_PROTOCOL) + return SymbolTrue def parse_from_pcl_file(definitions, pickle_file: str): From 61a44ae9bde10a7e8196047a62e100c77a096795 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 26 Apr 2026 10:39:32 -0400 Subject: [PATCH 3/3] Correct Sandboxed doc tagging --- mathics/builtin/files_io/files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mathics/builtin/files_io/files.py b/mathics/builtin/files_io/files.py index 642a1df07..0c12a0849 100644 --- a/mathics/builtin/files_io/files.py +++ b/mathics/builtin/files_io/files.py @@ -237,8 +237,9 @@ class DumpParse(Builtin): a Pickle file $output$. $True$ is returned if everthing want okay. - >S dumpParsedFile = FileNameJoin[{$TemporaryDirectory, "BoolEval.mx3"}] - >S DumpParse["BoolEval/BoolEval.m", dumpParsedFile] + S> dumpParsedFile = FileNameJoin[{$TemporaryDirectory, "BoolEval.mx3"}] + = ... + S> DumpParse["BoolEval/BoolEval.m", dumpParsedFile] = True #> Clear[dumpParsedFile] """