Add parsed files and read them...

mathics/builtin/files_io/files.py

@@ -40,7 +40,14 @@
 )
 from mathics.eval.directories import TMP_DIR
 from mathics.eval.encoding import CHARACTER_ENCODING_MAP
-from mathics.eval.files_io.files import eval_Close, eval_Get, eval_Open, eval_Read
+from mathics.eval.files_io.files import (
+    eval_Close,
+    eval_DumpParse,
+    eval_Get,
+    eval_Get_from_PCL,
+    eval_Open,
+    eval_Read,
+)
 from mathics.eval.files_io.read import (
     Mathics3Open,
     channel_to_stream,
@@ -220,6 +227,96 @@ def eval(self, obj, evaluation: Evaluation):
         return eval_Close(obj, evaluation)
 
 
+class DumpParse(Builtin):
+    """
+    ## <url>:trace native symbol:</url>
+
+    <dl>
+      <dt>'DumpParse'[$input$, $output$, $Options$]
+      <dd>Reads Mathics3 source text  $input$, parses it and write the the M-expressions \
+      a Pickle file $output$. $True$ is returned if everthing want okay.
+    </dl>
+
+    S> dumpParsedFile = FileNameJoin[{$TemporaryDirectory, "BoolEval.mx3"}]
+     = ...
+    S> DumpParse["BoolEval/BoolEval.m", dumpParsedFile]
+     = True
+    #> Clear[dumpParsedFile]
+    """
+
+    options = {
+        "CharacterEncoding": "Null",
+        "Path": "Null",
+        "Trace": "False",
+    }
+    summary_text = (
+        "Read and parse Mathics3 source text, and then write the parse to a PCL file"
+    )
+
+    def eval(
+        self, input: String, output: String, evaluation: Evaluation, options: dict
+    ):
+        "DumpParse[input_String, output_String, OptionsPattern[DumpParse]]"
+
+        # Make sure to pick up copy from module each time instead of using
+        # use "from ... import DEFAULT_TRACE_FN" which will not pick
+        # up run-time changes made to the module function.
+        trace_fn = io_files.DEFAULT_TRACE_FN
+
+        trace_get = evaluation.parse("Settings`$TraceGet")
+        if (
+            options["System`Trace"].to_python()
+            or trace_get.evaluate(evaluation) is SymbolTrue
+        ):
+            trace_fn = io_files.GET_PRINT_FN
+        # Process the "Path" option.
+        # The result will be put in py_path_directories
+        path_directories = options["System`Path"]
+        py_path_directories = None
+        if (
+            path_directories is not SymbolNull
+            and (py_path_directories := path_directories.to_python(string_quotes=False))
+            is not None
+        ):
+            if isinstance(py_path_directories, tuple):
+                for dir in py_path_directories:
+                    if not isinstance(dir, str):
+                        evaluation.message("DumpParse", "path", dir)
+                        py_path_directories = None
+                        break
+            elif isinstance(py_path_directories, str):
+                py_path_directories = [py_path_directories]
+            else:
+                evaluation.message("DumpParse", "path", path_directories)
+                py_path_directories = None
+
+        # Process the "CharacterEncoding" option.
+        encoding = options["System`CharacterEncoding"]
+        py_current_encoding = evaluation.definitions.get_ownvalue(
+            "System`$CharacterEncoding"
+        ).value
+        if isinstance(encoding, String):
+            py_encoding = encoding.to_python(string_quotes=False)
+            if py_encoding not in CHARACTER_ENCODING_MAP:
+                # "noopen" matches WMA. This is nonsensical.
+                evaluation.message("Get", "noopen", encoding)
+                py_encoding = py_current_encoding
+        else:
+            if encoding is not SymbolNull:
+                evaluation.message("$CharacterEncoding", "charcode", encoding)
+            py_encoding = py_current_encoding
+
+        # Dump perform the actual evaluation
+        return eval_DumpParse(
+            input.value,
+            output.value,
+            evaluation,
+            py_encoding,
+            trace_fn,
+            py_path_directories,
+        )
+
+
 class EndOfFile(Builtin):
     """
     <url>:WMA link:
@@ -418,7 +515,7 @@ def eval(self, path: String, evaluation: Evaluation, options: dict):
             if isinstance(py_path_directories, tuple):
                 for dir in py_path_directories:
                     if not isinstance(dir, str):
-                        evaluation.message("Put", "path", dir)
+                        evaluation.message("Get", "path", dir)
                         py_path_directories = None
                         break
             elif isinstance(py_path_directories, str):
@@ -444,6 +541,15 @@ def eval(self, path: String, evaluation: Evaluation, options: dict):
             py_encoding = py_current_encoding
 
         # perform the actual evaluation
+        if path.value.endswith("mx3"):
+            return eval_Get_from_PCL(
+                path.value,
+                evaluation,
+                py_encoding,
+                trace_fn,
+                py_path_directories,
+            )
+
         return eval_Get(
             path.value,
             evaluation,

mathics/core/parser/util.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import pickle
 from typing import FrozenSet, Optional, Tuple
 
 import mathics_scanner.location
@@ -8,14 +9,31 @@
 
 from mathics.core.definitions import Definitions
 from mathics.core.element import BaseElement
+from mathics.core.parser.ast import Node as ASTNode
 from mathics.core.parser.convert import convert
 from mathics.core.parser.feed import MathicsSingleLineFeeder
 from mathics.core.parser.parser import Parser
-from mathics.core.symbols import Symbol, ensure_context
+from mathics.core.symbols import Symbol, SymbolTrue, ensure_context
+from mathics.core.systemsymbols import SymbolFailed
 
 parser = Parser()
 
 
+def dump_exprs_to_pcl_file(exprs, pickle_file: str) -> Symbol:
+    """
+    Parse input from `feeder` and pickle serialize the parsed M-expression Python written
+    to pickle_file.
+    Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle.
+
+    Return SymbolTrue if things went okay.
+    """
+    # Open the file in binary write mode
+    with open(pickle_file, "wb") as f:
+        # Protocol -1 uses the highest available binary protocol for efficiency
+        pickle.dump(exprs, f, protocol=pickle.HIGHEST_PROTOCOL)
+    return SymbolTrue
+
+
 def parse(definitions, feeder: LineFeeder) -> Optional[BaseElement]:
     """
     Parse input (from the frontend, -e, input files, ToExpression etc).
@@ -85,6 +103,36 @@ def parse_returning_code(
     return converted, source_text
 
 
+def parse_dump_to_pcl_file(feeder: LineFeeder, pickle_file: str) -> Symbol:
+    """
+    Parse input from `feeder` and pickle serialize the parsed M-expression Python written
+    to pickle_file.
+    Serializes a Mathics3 AST Node to a file `pickle_file` using Python pickle.
+    """
+    ast = parser.parse(feeder)
+
+    if ast is None:
+        return SymbolFailed
+    # Ensure the input is actually a Node (optional safety check)
+    if not isinstance(ast, ASTNode):
+        raise TypeError(f"Expected mathics.core.parser.ast.Node, got {type(ast)}")
+
+    # Open the file in binary write mode
+    with open(pickle_file, "wb") as f:
+        # Protocol -1 uses the highest available binary protocol for efficiency
+        pickle.dump(ast, f, protocol=pickle.HIGHEST_PROTOCOL)
+    return SymbolTrue
+
+
+def parse_from_pcl_file(definitions, pickle_file: str):
+
+    with open(pickle_file, "rb") as f:
+        # Load the object from the binary file.
+        result = pickle.load(f)
+
+    return result
+
+
 class SystemDefinitions:
     """
     Dummy Definitions object that puts every unqualified symbol in

mathics/eval/files_io/files.py

@@ -23,9 +23,13 @@
 from mathics.core.evaluation import Evaluation
 from mathics.core.expression import BaseElement, Expression
 from mathics.core.parser import MathicsFileLineFeeder, MathicsMultiLineFeeder
-from mathics.core.parser.util import parse_incrementally_by_line
+from mathics.core.parser.util import (
+    dump_exprs_to_pcl_file,
+    parse_from_pcl_file,
+    parse_incrementally_by_line,
+)
 from mathics.core.streams import path_search, stream_manager
-from mathics.core.symbols import Symbol, SymbolNull
+from mathics.core.symbols import Symbol, SymbolNull, SymbolTrue
 from mathics.core.systemsymbols import (
     SymbolEndOfFile,
     SymbolExpression,
@@ -117,6 +121,84 @@ def eval_Close(obj, evaluation: Evaluation):
     return name
 
 
+def eval_DumpParse(
+    input_path: str,
+    output_path: str,
+    evaluation: Evaluation,
+    encoding: str,
+    trace_fn: Optional[Callable] = DEFAULT_TRACE_FN,
+    path_directories: Optional[Sequence[str]] = None,
+) -> Symbol:
+    """
+    Reads file `input_path`, parses each expression in the file
+    accumulating them in a list. This list is Python Pickled and
+    written to `output_path`. True is returned if everything succeeded.
+    """
+
+    if path_directories is None:
+        path_directories = tuple(streams.PATH_VAR)
+    resolved_path, _ = path_search(input_path, path_directories)
+    if resolved_path is None:
+        resolved_path = input_path
+    definitions = evaluation.definitions
+
+    # Wrap actual evaluation to handle setting $Input
+    # and $InputFileName
+    # store input paths of calling context
+
+    global INPUT_VAR
+    outer_input_var = INPUT_VAR
+    outer_inputfile = definitions.get_inputfile()
+
+    # Set a new input path.
+    INPUT_VAR = resolved_path
+    definitions.set_inputfile(INPUT_VAR)
+
+    # Save old PATH_VAR in case it gets changed in running Get?
+    # This seems to be needed, but not 100% sure there isn't
+    # a better and more robust way.
+    old_streams_path_var = streams.PATH_VAR
+    streams.PATH_VAR = SymbolPath.evaluate(evaluation).to_python(string_quotes=False)
+
+    queries = []
+
+    if trace_fn is not None:
+        trace_fn(0, resolved_path + "\n")
+    try:
+        with Mathics3Open(resolved_path, "r", encoding=encoding) as f:
+            feeder = MathicsFileLineFeeder(f, trace_fn)
+            while not feeder.empty():
+                try:
+                    # Note: we use mathics.core.parser.parse
+                    # so that tracing/debugging can intercept parse()
+                    query = mathics.core.parser.parse(definitions, feeder)
+                except SyntaxError:
+                    return SymbolNull
+                finally:
+                    feeder.send_messages(evaluation)
+                if query is None:  # blank line / comment
+                    continue
+                else:
+                    queries.append(query)
+
+                # result = query.evaluate(evaluation)
+    except IOError:
+        evaluation.message("DumpParse", "noopen", input_path)
+        return SymbolFailed
+    except MessageException as e:
+        e.message(evaluation)
+        return SymbolFailed
+    finally:
+        # Whether we had an exception or not, restore the input path
+        # and the state of definitions prior to calling Get.
+        INPUT_VAR = outer_input_var
+        definitions.set_inputfile(outer_inputfile)
+        streams.PATH_VAR = old_streams_path_var
+
+    dump_exprs_to_pcl_file(queries, output_path)
+    return SymbolTrue
+
+
 def eval_Get(
     path: str,
     evaluation: Evaluation,
@@ -186,6 +268,32 @@ def eval_Get(
     return result
 
 
+def eval_Get_from_PCL(
+    path: str,
+    evaluation: Evaluation,
+    encoding: str,
+    trace_fn: Optional[Callable] = DEFAULT_TRACE_FN,
+    path_directories: Optional[Sequence[str]] = None,
+):
+    """
+    Reads a file and evaluates each expression, returning only the last one.
+    """
+
+    result = None
+    if path_directories is None:
+        path_directories = tuple(streams.PATH_VAR)
+    resolved_path, _ = path_search(path, path_directories)
+    if resolved_path is None:
+        resolved_path = path
+
+    parse_list = parse_from_pcl_file(evaluation.definitions, resolved_path)
+    if not isinstance(parse_list, list):
+        return None
+    for query in parse_list:
+        result = query.evaluate(evaluation)
+    return result
+
+
 def eval_Open(
     name: String,
     mode: str,