diff --git a/mathics/builtin/files_io/files.py b/mathics/builtin/files_io/files.py
index 7245d9242..0c12a0849 100644
--- a/mathics/builtin/files_io/files.py
+++ b/mathics/builtin/files_io/files.py
@@ -40,7 +40,14 @@
)
from mathics.eval.directories import TMP_DIR
from mathics.eval.encoding import CHARACTER_ENCODING_MAP
-from mathics.eval.files_io.files import eval_Close, eval_Get, eval_Open, eval_Read
+from mathics.eval.files_io.files import (
+ eval_Close,
+ eval_DumpParse,
+ eval_Get,
+ eval_Get_from_PCL,
+ eval_Open,
+ eval_Read,
+)
from mathics.eval.files_io.read import (
Mathics3Open,
channel_to_stream,
@@ -220,6 +227,96 @@ def eval(self, obj, evaluation: Evaluation):
return eval_Close(obj, evaluation)
+class DumpParse(Builtin):
+ """
+ ## :trace native symbol:
+
+
+ - 'DumpParse'[$input$, $output$, $Options$]
+
- Reads Mathics3 source text $input$, parses it and write the the M-expressions \
+ a Pickle file $output$. $True$ is returned if everthing want okay.
+
+
+ S> dumpParsedFile = FileNameJoin[{$TemporaryDirectory, "BoolEval.mx3"}]
+ = ...
+ S> DumpParse["BoolEval/BoolEval.m", dumpParsedFile]
+ = True
+ #> Clear[dumpParsedFile]
+ """
+
+ options = {
+ "CharacterEncoding": "Null",
+ "Path": "Null",
+ "Trace": "False",
+ }
+ summary_text = (
+ "Read and parse Mathics3 source text, and then write the parse to a PCL file"
+ )
+
+ def eval(
+ self, input: String, output: String, evaluation: Evaluation, options: dict
+ ):
+ "DumpParse[input_String, output_String, OptionsPattern[DumpParse]]"
+
+ # Make sure to pick up copy from module each time instead of using
+ # use "from ... import DEFAULT_TRACE_FN" which will not pick
+ # up run-time changes made to the module function.
+ trace_fn = io_files.DEFAULT_TRACE_FN
+
+ trace_get = evaluation.parse("Settings`$TraceGet")
+ if (
+ options["System`Trace"].to_python()
+ or trace_get.evaluate(evaluation) is SymbolTrue
+ ):
+ trace_fn = io_files.GET_PRINT_FN
+ # Process the "Path" option.
+ # The result will be put in py_path_directories
+ path_directories = options["System`Path"]
+ py_path_directories = None
+ if (
+ path_directories is not SymbolNull
+ and (py_path_directories := path_directories.to_python(string_quotes=False))
+ is not None
+ ):
+ if isinstance(py_path_directories, tuple):
+ for dir in py_path_directories:
+ if not isinstance(dir, str):
+ evaluation.message("DumpParse", "path", dir)
+ py_path_directories = None
+ break
+ elif isinstance(py_path_directories, str):
+ py_path_directories = [py_path_directories]
+ else:
+ evaluation.message("DumpParse", "path", path_directories)
+ py_path_directories = None
+
+ # Process the "CharacterEncoding" option.
+ encoding = options["System`CharacterEncoding"]
+ py_current_encoding = evaluation.definitions.get_ownvalue(
+ "System`$CharacterEncoding"
+ ).value
+ if isinstance(encoding, String):
+ py_encoding = encoding.to_python(string_quotes=False)
+ if py_encoding not in CHARACTER_ENCODING_MAP:
+ # "noopen" matches WMA. This is nonsensical.
+ evaluation.message("Get", "noopen", encoding)
+ py_encoding = py_current_encoding
+ else:
+ if encoding is not SymbolNull:
+ evaluation.message("$CharacterEncoding", "charcode", encoding)
+ py_encoding = py_current_encoding
+
+ # Dump perform the actual evaluation
+ return eval_DumpParse(
+ input.value,
+ output.value,
+ evaluation,
+ py_encoding,
+ trace_fn,
+ py_path_directories,
+ )
+
+
class EndOfFile(Builtin):
"""
:WMA link:
@@ -418,7 +515,7 @@ def eval(self, path: String, evaluation: Evaluation, options: dict):
if isinstance(py_path_directories, tuple):
for dir in py_path_directories:
if not isinstance(dir, str):
- evaluation.message("Put", "path", dir)
+ evaluation.message("Get", "path", dir)
py_path_directories = None
break
elif isinstance(py_path_directories, str):
@@ -444,6 +541,15 @@ def eval(self, path: String, evaluation: Evaluation, options: dict):
py_encoding = py_current_encoding
# perform the actual evaluation
+ if path.value.endswith("mx3"):
+ return eval_Get_from_PCL(
+ path.value,
+ evaluation,
+ py_encoding,
+ trace_fn,
+ py_path_directories,
+ )
+
return eval_Get(
path.value,
evaluation,
diff --git a/mathics/core/parser/util.py b/mathics/core/parser/util.py
index ece35aa21..770641677 100644
--- a/mathics/core/parser/util.py
+++ b/mathics/core/parser/util.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
+import pickle
from typing import FrozenSet, Optional, Tuple
import mathics_scanner.location
@@ -8,14 +9,31 @@
from mathics.core.definitions import Definitions
from mathics.core.element import BaseElement
+from mathics.core.parser.ast import Node as ASTNode
from mathics.core.parser.convert import convert
from mathics.core.parser.feed import MathicsSingleLineFeeder
from mathics.core.parser.parser import Parser
-from mathics.core.symbols import Symbol, ensure_context
+from mathics.core.symbols import Symbol, SymbolTrue, ensure_context
+from mathics.core.systemsymbols import SymbolFailed
parser = Parser()
+def dump_exprs_to_pcl_file(exprs, pickle_file: str) -> Symbol:
+ """
+ Parse input from `feeder` and pickle serialize the parsed M-expression Python written
+ to pickle_file.
+ Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle.
+
+ Return SymbolTrue if things went okay.
+ """
+ # Open the file in binary write mode
+ with open(pickle_file, "wb") as f:
+ # Protocol -1 uses the highest available binary protocol for efficiency
+ pickle.dump(exprs, f, protocol=pickle.HIGHEST_PROTOCOL)
+ return SymbolTrue
+
+
def parse(definitions, feeder: LineFeeder) -> Optional[BaseElement]:
"""
Parse input (from the frontend, -e, input files, ToExpression etc).
@@ -85,6 +103,36 @@ def parse_returning_code(
return converted, source_text
+def parse_dump_to_pcl_file(feeder: LineFeeder, pickle_file: str) -> Symbol:
+ """
+ Parse input from `feeder` and pickle serialize the parsed M-expression Python written
+ to pickle_file.
+ Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle.
+ """
+ ast = parser.parse(feeder)
+
+ if ast is None:
+ return SymbolFailed
+ # Ensure the input is actually a Node (optional safety check)
+ if not isinstance(ast, ASTNode):
+ raise TypeError(f"Expected mathics.core.parser.ast.Node, got {type(ast)}")
+
+ # Open the file in binary write mode
+ with open(pickle_file, "wb") as f:
+ # Protocol -1 uses the highest available binary protocol for efficiency
+ pickle.dump(ast, f, protocol=pickle.HIGHEST_PROTOCOL)
+ return SymbolTrue
+
+
+def parse_from_pcl_file(definitions, pickle_file: str):
+
+ with open(pickle_file, "rb") as f:
+ # Load the object from the binary file.
+ result = pickle.load(f)
+
+ return result
+
+
class SystemDefinitions:
"""
Dummy Definitions object that puts every unqualified symbol in
diff --git a/mathics/eval/files_io/files.py b/mathics/eval/files_io/files.py
index a417ec66c..5560d95a9 100644
--- a/mathics/eval/files_io/files.py
+++ b/mathics/eval/files_io/files.py
@@ -23,9 +23,13 @@
from mathics.core.evaluation import Evaluation
from mathics.core.expression import BaseElement, Expression
from mathics.core.parser import MathicsFileLineFeeder, MathicsMultiLineFeeder
-from mathics.core.parser.util import parse_incrementally_by_line
+from mathics.core.parser.util import (
+ dump_exprs_to_pcl_file,
+ parse_from_pcl_file,
+ parse_incrementally_by_line,
+)
from mathics.core.streams import path_search, stream_manager
-from mathics.core.symbols import Symbol, SymbolNull
+from mathics.core.symbols import Symbol, SymbolNull, SymbolTrue
from mathics.core.systemsymbols import (
SymbolEndOfFile,
SymbolExpression,
@@ -117,6 +121,84 @@ def eval_Close(obj, evaluation: Evaluation):
return name
+def eval_DumpParse(
+ input_path: str,
+ output_path: str,
+ evaluation: Evaluation,
+ encoding: str,
+ trace_fn: Optional[Callable] = DEFAULT_TRACE_FN,
+ path_directories: Optional[Sequence[str]] = None,
+) -> Symbol:
+ """
+ Reads file `input_path`, parses each expression in the file
+ accumulating them in a list. This list is Python Pickled and
+ written to `output_path`. True is returned if everything succeeded.
+ """
+
+ if path_directories is None:
+ path_directories = tuple(streams.PATH_VAR)
+ resolved_path, _ = path_search(input_path, path_directories)
+ if resolved_path is None:
+ resolved_path = input_path
+ definitions = evaluation.definitions
+
+ # Wrap actual evaluation to handle setting $Input
+ # and $InputFileName
+ # store input paths of calling context
+
+ global INPUT_VAR
+ outer_input_var = INPUT_VAR
+ outer_inputfile = definitions.get_inputfile()
+
+ # Set a new input path.
+ INPUT_VAR = resolved_path
+ definitions.set_inputfile(INPUT_VAR)
+
+ # Save old PATH_VAR in case it gets changed in running Get?
+ # This seems to be needed, but not 100% sure there isn't
+ # a better and more robust way.
+ old_streams_path_var = streams.PATH_VAR
+ streams.PATH_VAR = SymbolPath.evaluate(evaluation).to_python(string_quotes=False)
+
+ queries = []
+
+ if trace_fn is not None:
+ trace_fn(0, resolved_path + "\n")
+ try:
+ with Mathics3Open(resolved_path, "r", encoding=encoding) as f:
+ feeder = MathicsFileLineFeeder(f, trace_fn)
+ while not feeder.empty():
+ try:
+ # Note: we use mathics.core.parser.parse
+ # so that tracing/debugging can intercept parse()
+ query = mathics.core.parser.parse(definitions, feeder)
+ except SyntaxError:
+ return SymbolNull
+ finally:
+ feeder.send_messages(evaluation)
+ if query is None: # blank line / comment
+ continue
+ else:
+ queries.append(query)
+
+ # result = query.evaluate(evaluation)
+ except IOError:
+ evaluation.message("DumpParse", "noopen", input_path)
+ return SymbolFailed
+ except MessageException as e:
+ e.message(evaluation)
+ return SymbolFailed
+ finally:
+ # Whether we had an exception or not, restore the input path
+ # and the state of definitions prior to calling Get.
+ INPUT_VAR = outer_input_var
+ definitions.set_inputfile(outer_inputfile)
+ streams.PATH_VAR = old_streams_path_var
+
+ dump_exprs_to_pcl_file(queries, output_path)
+ return SymbolTrue
+
+
def eval_Get(
path: str,
evaluation: Evaluation,
@@ -186,6 +268,32 @@ def eval_Get(
return result
+def eval_Get_from_PCL(
+ path: str,
+ evaluation: Evaluation,
+ encoding: str,
+ trace_fn: Optional[Callable] = DEFAULT_TRACE_FN,
+ path_directories: Optional[Sequence[str]] = None,
+):
+ """
+ Reads a file and evaluates each expression, returning only the last one.
+ """
+
+ result = None
+ if path_directories is None:
+ path_directories = tuple(streams.PATH_VAR)
+ resolved_path, _ = path_search(path, path_directories)
+ if resolved_path is None:
+ resolved_path = path
+
+ parse_list = parse_from_pcl_file(evaluation.definitions, resolved_path)
+ if not isinstance(parse_list, list):
+ return None
+ for query in parse_list:
+ result = query.evaluate(evaluation)
+ return result
+
+
def eval_Open(
name: String,
mode: str,