From 108749dda54b944ee15983be2222061d79125106 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Mon, 4 May 2026 11:15:10 -0500 Subject: [PATCH 1/2] Make tox factor usage more extensible Previously, the test environments were binary flagged between `mindeps` and `!mindeps`. This is unfortunately bad for adding additional factors, as discovered when testing `orjson` as a factor. Because `tox` does not allow negative factors to be layered with "OR" semantics, it becomes difficult to deselect the locked requirements in `test.txt` in order to replace them. The base testenv is simplified to have unconditional dependencies, and factor-based dependency selection is done in descendant envs which can therefore fully replace the deps declared by the base env. The change in configuration also provides an opportunity to refactor the `depends` declaration into the list form which makes it inheritable. --- tox.ini | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 5f74c0ae..c842d66e 100644 --- a/tox.ini +++ b/tox.ini @@ -18,31 +18,36 @@ labels = # build a wheel, not a tarball, and use a common env to do it (so that the wheel is shared) package = wheel wheel_build_env = build_wheel +deps = -r requirements/py{py_dot_ver}/test.txt +commands = coverage run -m pytest {posargs} +depends = coverage_clean,lint +[testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-mindeps] +deps = -r requirements/py{py_dot_ver}/test-mindeps.txt + +[testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-sphinxext] deps = - !mindeps: -r requirements/py{py_dot_ver}/test.txt - mindeps: -r requirements/py{py_dot_ver}/test-mindeps.txt - sphinxext: -r requirements/py{py_dot_ver}/docs.txt -commands = coverage run -m pytest {posargs} -depends = - py{3.14,3.13,3.12,3.11,3.10,3.9}{-mindeps,-sphinxext,}: coverage_clean, lint - coverage_report: py{3.14,3.13,3.12,3.11,3.10,3.9}{-mindeps,-sphinxext,} + -r requirements/py{py_dot_ver}/test.txt + -r requirements/py{py_dot_ver}/docs.txt [testenv:coverage_clean] dependency_groups = coverage skip_install = true commands = coverage erase +depends = [testenv:coverage_report] dependency_groups = coverage skip_install = true commands_pre = -coverage combine commands = coverage report --skip-covered +depends = py{3.14,3.13,3.12,3.11,3.10,3.9}{-mindeps,-sphinxext,} [testenv:lint] deps = pre-commit skip_install = true commands = pre-commit run --all-files +depends = [testenv:mypy,mypy-{py3.9,py3.14}] deps = -r requirements/py{py_dot_ver}/typing.txt From 42e39991836b40cd78bf6023549b5d2b5c7007ff Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Fri, 1 May 2026 18:07:29 -0500 Subject: [PATCH 2/2] Support `orjson` as an alternative encoder/decoder Add support for user selection of `orjson` as a faster and more correct JSON encoder/decoder. Request encoding is done by sending encoded bytes through `requests` (which is supported), and request decoding is done via a helper which reads `requests.Response.content` (bytes). This is primarily applied in - the JSON request encoder - the Response class - the APIError base class Additionally, GlobusApp and Transfer retry policies decode response contents, and are here updated to load from bytes. `mypy` runs are expanded to include `orjson` as a dependency, and test runs include it via a factor. New tests can directly target the encoder class, but as there is no central point of control for decoding, there is no clearly testable element for that purpose. tox configuration and testing requirements are updated to pull in `orjson` when the `-orjson` factor is used. GitHub Actions testing is now expanded to include `-orjson` tests. --- .github/workflows/test.yaml | 4 ++ ...1_170758_sirosen_define_orjson_encoder.rst | 8 +++ pyproject.toml | 3 + requirements/py3.10/test-orjson.txt | 55 +++++++++++++++++ requirements/py3.10/typing.txt | 34 ++++++----- requirements/py3.11/test-orjson.txt | 47 +++++++++++++++ requirements/py3.11/typing.txt | 38 ++++++------ requirements/py3.12/test-orjson.txt | 47 +++++++++++++++ requirements/py3.12/typing.txt | 38 ++++++------ requirements/py3.13/test-orjson.txt | 47 +++++++++++++++ requirements/py3.13/typing.txt | 38 ++++++------ requirements/py3.14/test-orjson.txt | 47 +++++++++++++++ requirements/py3.14/typing.txt | 38 ++++++------ requirements/py3.9/test-orjson.txt | 59 +++++++++++++++++++ requirements/py3.9/typing.txt | 34 ++++++----- scripts/ensure_min_python_is_tested.py | 2 +- src/globus_sdk/_internal/orjson_compat.py | 45 ++++++++++++++ src/globus_sdk/config/__init__.py | 8 ++- src/globus_sdk/config/env_vars.py | 15 +++++ src/globus_sdk/exc/api.py | 6 +- .../experimental/transfer_v2/transport.py | 5 +- src/globus_sdk/globus_app/app.py | 6 +- src/globus_sdk/response.py | 6 +- src/globus_sdk/services/transfer/transport.py | 5 +- src/globus_sdk/testing/registry.py | 1 + src/globus_sdk/transport/encoders.py | 33 ++++++++--- tests/unit/test_config.py | 16 +++++ .../unit/transport/test_transport_encoders.py | 34 +++++++++++ tox.ini | 7 +++ 29 files changed, 612 insertions(+), 114 deletions(-) create mode 100644 changelog.d/20260501_170758_sirosen_define_orjson_encoder.rst create mode 100644 requirements/py3.10/test-orjson.txt create mode 100644 requirements/py3.11/test-orjson.txt create mode 100644 requirements/py3.12/test-orjson.txt create mode 100644 requirements/py3.13/test-orjson.txt create mode 100644 requirements/py3.14/test-orjson.txt create mode 100644 requirements/py3.9/test-orjson.txt create mode 100644 src/globus_sdk/_internal/orjson_compat.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 88779cf2..591ebbf0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,6 +44,8 @@ jobs: - "3.14" tox-post-environments: - "py3.9-mindeps" + - "py3.9-orjson" + - "py3.14-orjson" - "py3.11-sphinxext" - "coverage_report" @@ -53,6 +55,7 @@ jobs: - "3.11" tox-post-environments: - "py3.11-sphinxext" + - "py3.14-orjson" - "coverage_report" - name: "Windows" @@ -62,6 +65,7 @@ jobs: - "3.11" tox-post-environments: - "py3.9-mindeps" + - "py3.14-orjson" - "py3.11-sphinxext" - "coverage_report" diff --git a/changelog.d/20260501_170758_sirosen_define_orjson_encoder.rst b/changelog.d/20260501_170758_sirosen_define_orjson_encoder.rst new file mode 100644 index 00000000..82ad0184 --- /dev/null +++ b/changelog.d/20260501_170758_sirosen_define_orjson_encoder.rst @@ -0,0 +1,8 @@ +Added +----- + +- The SDK now supports use of ``orjson`` as an alternative JSON encoder and decoder. + When ``GLOBUS_SDK_PREFER_ORJSON=1`` is set, request sending and response decoding + will prefer to use ``orjson`` if it is installed and available, gracefully failing + over to the standard library ``json`` module if it is not. This setting will become + the default behavior in a future major version of the SDK. (:pr:`NUMBER`) diff --git a/pyproject.toml b/pyproject.toml index 3b970ce3..204975b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ docs = [ "scriv", ] coverage = ["coverage[toml]"] +orjson = ["orjson>=3"] test = [ {include-group = "coverage"}, "pytest", "pytest-xdist", "pytest-randomly", "flaky", @@ -83,6 +84,8 @@ typing = [ "responses", # similarly, sphinx is needed to type-check our sphinx extension "sphinx", + # include any optional test deps + {include-group = "orjson"}, ] typing-mindeps = [ {include-group = "typing"}, diff --git a/requirements/py3.10/test-orjson.txt b/requirements/py3.10/test-orjson.txt new file mode 100644 index 00000000..e87fc173 --- /dev/null +++ b/requirements/py3.10/test-orjson.txt @@ -0,0 +1,55 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +exceptiongroup==1.3.1 + # via pytest +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +tomli==2.4.1 + # via + # coverage + # pytest +typing-extensions==4.15.0 + # via exceptiongroup +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.10/typing.txt b/requirements/py3.10/typing.txt index fb57ddad..51b712ae 100644 --- a/requirements/py3.10/typing.txt +++ b/requirements/py3.10/typing.txt @@ -6,39 +6,43 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests docutils==0.21.2 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in snowballstemmer==3.0.1 # via sphinx @@ -56,23 +60,23 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -tomli==2.3.0 +tomli==2.4.1 # via # mypy # sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.11/test-orjson.txt b/requirements/py3.11/test-orjson.txt new file mode 100644 index 00000000..9f33bfdf --- /dev/null +++ b/requirements/py3.11/test-orjson.txt @@ -0,0 +1,47 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.11/typing.txt b/requirements/py3.11/typing.txt index 45bbb807..6b278553 100644 --- a/requirements/py3.11/typing.txt +++ b/requirements/py3.11/typing.txt @@ -6,45 +6,49 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests -docutils==0.21.2 +docutils==0.22.4 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 # via sphinx -sphinx==8.2.3 +sphinx==9.0.4 # via -r .typing.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -60,17 +64,17 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.12/test-orjson.txt b/requirements/py3.12/test-orjson.txt new file mode 100644 index 00000000..6e172c58 --- /dev/null +++ b/requirements/py3.12/test-orjson.txt @@ -0,0 +1,47 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.12/typing.txt b/requirements/py3.12/typing.txt index 89ab3466..cb35ef7c 100644 --- a/requirements/py3.12/typing.txt +++ b/requirements/py3.12/typing.txt @@ -6,45 +6,49 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests -docutils==0.21.2 +docutils==0.22.4 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 # via sphinx -sphinx==8.2.3 +sphinx==9.1.0 # via -r .typing.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -60,17 +64,17 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.13/test-orjson.txt b/requirements/py3.13/test-orjson.txt new file mode 100644 index 00000000..69e5c816 --- /dev/null +++ b/requirements/py3.13/test-orjson.txt @@ -0,0 +1,47 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.13/typing.txt b/requirements/py3.13/typing.txt index 06ba2a2c..ddb34548 100644 --- a/requirements/py3.13/typing.txt +++ b/requirements/py3.13/typing.txt @@ -6,45 +6,49 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests -docutils==0.21.2 +docutils==0.22.4 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 # via sphinx -sphinx==8.2.3 +sphinx==9.1.0 # via -r .typing.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -60,17 +64,17 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.14/test-orjson.txt b/requirements/py3.14/test-orjson.txt new file mode 100644 index 00000000..1ae899e6 --- /dev/null +++ b/requirements/py3.14/test-orjson.txt @@ -0,0 +1,47 @@ +# +# This file is autogenerated by pip-compile with Python 3.14 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.14/typing.txt b/requirements/py3.14/typing.txt index ec731f1f..b371ae76 100644 --- a/requirements/py3.14/typing.txt +++ b/requirements/py3.14/typing.txt @@ -6,45 +6,49 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests -docutils==0.21.2 +docutils==0.22.4 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 # via sphinx -sphinx==8.2.3 +sphinx==9.1.0 # via -r .typing.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -60,17 +64,17 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.9/test-orjson.txt b/requirements/py3.9/test-orjson.txt new file mode 100644 index 00000000..a1c033c6 --- /dev/null +++ b/requirements/py3.9/test-orjson.txt @@ -0,0 +1,59 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.10.7 + # via -r .test.in +exceptiongroup==1.3.1 + # via pytest +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +importlib-metadata==8.7.1 + # via pytest-randomly +iniconfig==2.1.0 + # via pytest +orjson==3.11.5 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==8.4.2 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.0.1 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.32.5 + # via responses +responses==0.26.0 + # via -r .test.in +tomli==2.4.1 + # via + # coverage + # pytest +typing-extensions==4.15.0 + # via exceptiongroup +urllib3==2.6.3 + # via + # requests + # responses +zipp==3.23.1 + # via importlib-metadata diff --git a/requirements/py3.9/typing.txt b/requirements/py3.9/typing.txt index 1dc4046b..6ed3d898 100644 --- a/requirements/py3.9/typing.txt +++ b/requirements/py3.9/typing.txt @@ -6,33 +6,37 @@ # alabaster==0.7.16 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests docutils==0.21.2 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==1.5.0 # via sphinx -importlib-metadata==8.7.0 +importlib-metadata==8.7.1 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.19.1 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.5 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses @@ -40,7 +44,7 @@ requests==2.32.5 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in snowballstemmer==3.0.1 # via sphinx @@ -58,7 +62,7 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -tomli==2.3.0 +tomli==2.4.1 # via # mypy # sphinx @@ -68,16 +72,16 @@ types-docutils==0.22.3.20251115 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.32.4.20260107 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses # types-requests -zipp==3.23.0 +zipp==3.23.1 # via importlib-metadata diff --git a/scripts/ensure_min_python_is_tested.py b/scripts/ensure_min_python_is_tested.py index f785fed1..a3f237a3 100644 --- a/scripts/ensure_min_python_is_tested.py +++ b/scripts/ensure_min_python_is_tested.py @@ -24,7 +24,7 @@ else: raise ValueError("Could not find 'Linux' in the test matrix.") - for environment in include["tox-post-environments"]: + for environment in include["tox-environments"]: if environment.endswith("-mindeps"): break else: diff --git a/src/globus_sdk/_internal/orjson_compat.py b/src/globus_sdk/_internal/orjson_compat.py new file mode 100644 index 00000000..6d7500a2 --- /dev/null +++ b/src/globus_sdk/_internal/orjson_compat.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import json +import typing as t + +from globus_sdk import config + +if t.TYPE_CHECKING: + import requests + +dumps: t.Callable[[t.Any], bytes] +loads: t.Callable[[str | bytes], t.Any] +try: + import orjson + + ORJSON_AVAILABLE: bool = True +except ImportError: + ORJSON_AVAILABLE = False + + def dumps(obj: t.Any) -> bytes: + return json.dumps(obj).encode() + + def loads(data: str | bytes) -> t.Any: + return json.loads(data) + +else: + dumps = orjson.dumps + loads = orjson.loads + + +def get_response_loader() -> t.Callable[[requests.Response], t.Any]: + # IMPORTANT: getting the config setting can error, so this function + # must be called outside of any error handling context for reading the response data + # which would catch ValueErrors + if ORJSON_AVAILABLE and config.get_prefer_orjson(): + + def loader(r: requests.Response) -> t.Any: + return loads(r.content) + + else: + + def loader(r: requests.Response) -> t.Any: + return r.json() + + return loader diff --git a/src/globus_sdk/config/__init__.py b/src/globus_sdk/config/__init__.py index da526811..fafb8b95 100644 --- a/src/globus_sdk/config/__init__.py +++ b/src/globus_sdk/config/__init__.py @@ -1,4 +1,9 @@ -from .env_vars import get_environment_name, get_http_timeout, get_ssl_verify +from .env_vars import ( + get_environment_name, + get_http_timeout, + get_prefer_orjson, + get_ssl_verify, +) from .environments import EnvConfig, get_service_url, get_webapp_url __all__ = ( @@ -6,6 +11,7 @@ "get_environment_name", "get_ssl_verify", "get_http_timeout", + "get_prefer_orjson", "get_service_url", "get_webapp_url", ) diff --git a/src/globus_sdk/config/env_vars.py b/src/globus_sdk/config/env_vars.py index 8e34958f..2a9c5b6b 100644 --- a/src/globus_sdk/config/env_vars.py +++ b/src/globus_sdk/config/env_vars.py @@ -17,6 +17,7 @@ ENVNAME_VAR = "GLOBUS_SDK_ENVIRONMENT" HTTP_TIMEOUT_VAR = "GLOBUS_SDK_HTTP_TIMEOUT" SSL_VERIFY_VAR = "GLOBUS_SDK_VERIFY_SSL" +PREFER_ORJSON_VAR = "GLOBUS_SDK_PREFER_ORJSON" def get_environment_name(inputenv: str | None = None) -> str: @@ -56,6 +57,11 @@ def get_http_timeout(value: float | None = None) -> float | None: return result +def get_prefer_orjson() -> bool: + var = os.getenv(PREFER_ORJSON_VAR, "false") + return _bool_cast(var) + + def _ssl_verify_cast(value: t.Any) -> bool | str: if isinstance(value, bool): return value @@ -83,3 +89,12 @@ def _float_cast(value: str) -> float: except ValueError as e: log.error(f'Value "{value}" can\'t cast to float') raise ValueError(f"Invalid config float: {value}") from e + + +def _bool_cast(value: str) -> bool: + if isinstance(value, str): + if value.lower() in {"y", "yes", "t", "true", "on", "1"}: + return True + if value.lower() in {"n", "no", "f", "false", "off", "0"}: + return False + raise ValueError(f"Invalid config bool: {value}") diff --git a/src/globus_sdk/exc/api.py b/src/globus_sdk/exc/api.py index 393293dc..3c91d370 100644 --- a/src/globus_sdk/exc/api.py +++ b/src/globus_sdk/exc/api.py @@ -6,7 +6,7 @@ import textwrap import typing as t -from globus_sdk._internal import guards +from globus_sdk._internal import guards, orjson_compat from .base import GlobusError from .err_info import ErrorInfoContainer @@ -132,11 +132,13 @@ def raw_json(self) -> dict[str, t.Any] | None: if self._cached_raw_json == _CACHE_SENTINEL: self._cached_raw_json = None if self._json_mimetype(): + response_loader = orjson_compat.get_response_loader() + try: # technically, this could be a non-dict JSON type, like a list or # string but in those cases the user can just cast -- the "normal" # case is a dict - self._cached_raw_json = self._underlying_response.json() + self._cached_raw_json = response_loader(self._underlying_response) except ValueError: log.error( "Error body could not be JSON decoded! " diff --git a/src/globus_sdk/experimental/transfer_v2/transport.py b/src/globus_sdk/experimental/transfer_v2/transport.py index c1860997..8f845dde 100644 --- a/src/globus_sdk/experimental/transfer_v2/transport.py +++ b/src/globus_sdk/experimental/transfer_v2/transport.py @@ -5,6 +5,7 @@ from __future__ import annotations +from globus_sdk._internal import orjson_compat from globus_sdk.transport import RetryCheck, RetryCheckResult, RetryContext from globus_sdk.transport.default_retry_checks import ( DEFAULT_RETRY_CHECKS, @@ -28,10 +29,12 @@ def check_transfer_v2_transient_error(ctx: RetryContext) -> RetryCheckResult: if ctx.response is not None and ( ctx.response.status_code in retry_config.transient_error_status_codes ): + response_loader = orjson_compat.get_response_loader() + try: # if any of the error objects have a `code` of ExternalError or # EndpointError then the error likely isn't transient - errors = ctx.response.json()["errors"] + errors = response_loader(ctx.response)["errors"] for error in errors: if error["code"] in ("ExternalError", "EndpointError"): return RetryCheckResult.no_decision diff --git a/src/globus_sdk/globus_app/app.py b/src/globus_sdk/globus_app/app.py index fc734804..6e8fff29 100644 --- a/src/globus_sdk/globus_app/app.py +++ b/src/globus_sdk/globus_app/app.py @@ -18,6 +18,7 @@ GlobusSDKUsageError, IDTokenDecoder, ) +from globus_sdk._internal import orjson_compat from globus_sdk._internal.type_definitions import Closable from globus_sdk.authorizers import GlobusAuthorizer from globus_sdk.gare import GARE, GlobusAuthorizationParameters, to_gare @@ -594,8 +595,11 @@ def _load_response_gare(response: Response | None) -> GARE | None: """Return a parsed GARE from a 403 response or None if not possible.""" if response is None or response.status_code != 403: return None + + response_loader = orjson_compat.get_response_loader() + try: - decoded_body = response.json() + decoded_body = response_loader(response) except JSONDecodeError: return None else: diff --git a/src/globus_sdk/response.py b/src/globus_sdk/response.py index 01d47f59..d373a262 100644 --- a/src/globus_sdk/response.py +++ b/src/globus_sdk/response.py @@ -6,7 +6,7 @@ import typing as t from functools import cached_property -from globus_sdk._internal import guards +from globus_sdk._internal import guards, orjson_compat log = logging.getLogger(__name__) @@ -78,8 +78,10 @@ def _parsed_json(self) -> t.Any: return self._wrapped._parsed_json if self._response is not None: + response_loader = orjson_compat.get_response_loader() + try: - return self._response.json() + return response_loader(self._response) except ValueError: log.warning("response data did not parse as JSON, data=None") return None diff --git a/src/globus_sdk/services/transfer/transport.py b/src/globus_sdk/services/transfer/transport.py index 4ab2c688..0f9ddcc7 100644 --- a/src/globus_sdk/services/transfer/transport.py +++ b/src/globus_sdk/services/transfer/transport.py @@ -5,6 +5,7 @@ from __future__ import annotations +from globus_sdk._internal import orjson_compat from globus_sdk.transport import RetryCheck, RetryCheckResult, RetryContext from globus_sdk.transport.default_retry_checks import ( DEFAULT_RETRY_CHECKS, @@ -25,8 +26,10 @@ def check_transfer_transient_error(ctx: RetryContext) -> RetryCheckResult: if ctx.response is not None and ( ctx.response.status_code in retry_config.transient_error_status_codes ): + response_loader = orjson_compat.get_response_loader() + try: - code = ctx.response.json()["code"] + code = response_loader(ctx.response)["code"] except (ValueError, KeyError): code = "" diff --git a/src/globus_sdk/testing/registry.py b/src/globus_sdk/testing/registry.py index 54bd47af..b6fca8c5 100644 --- a/src/globus_sdk/testing/registry.py +++ b/src/globus_sdk/testing/registry.py @@ -7,6 +7,7 @@ import responses import globus_sdk +import globus_sdk.experimental from .models import RegisteredResponse, ResponseList, ResponseSet diff --git a/src/globus_sdk/transport/encoders.py b/src/globus_sdk/transport/encoders.py index 5052f47e..03c4876a 100644 --- a/src/globus_sdk/transport/encoders.py +++ b/src/globus_sdk/transport/encoders.py @@ -6,6 +6,8 @@ import requests +from globus_sdk import config +from globus_sdk._internal import orjson_compat from globus_sdk._missing import MISSING, filter_missing @@ -105,6 +107,13 @@ class JSONRequestEncoder(RequestEncoder): that APIs requiring a content-type of "application/json" are able to read the data. """ + def __init__(self, *, use_orjson: bool | None = None) -> None: + self.use_orjson = ( + use_orjson + if use_orjson is not None + else (orjson_compat.ORJSON_AVAILABLE and config.get_prefer_orjson()) + ) + def encode( self, method: str, @@ -115,13 +124,23 @@ def encode( ) -> requests.Request: if data is not None: headers = {"Content-Type": "application/json", **headers} - return requests.Request( - method, - url, - json=self._prepare_data(data), - params=self._prepare_params(params), - headers=self._prepare_headers(headers), - ) + + if self.use_orjson: + return requests.Request( + method, + url, + data=orjson_compat.dumps(self._prepare_data(data)), + params=self._prepare_params(params), + headers=self._prepare_headers(headers), + ) + else: + return requests.Request( + method, + url, + json=self._prepare_data(data), + params=self._prepare_params(params), + headers=self._prepare_headers(headers), + ) class FormRequestEncoder(RequestEncoder): diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 06b506c9..0f62b05b 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -211,3 +211,19 @@ def test_service_url_from_env_var(): globus_sdk.config.get_webapp_url() == f"https://app.{env}.globuscs.info/" ) + + +@pytest.mark.parametrize( + "value, expected_result", + [(x, True) for x in ["1", "YES", "true", "t", "True", "ON"]] + + [(x, False) for x in ["0", "NO", "false", "f", "False", "OFF"]] + + [("invalid", ValueError), ("1.0", ValueError)], +) +def test_get_prefer_orjson(value, expected_result, monkeypatch): + monkeypatch.setenv("GLOBUS_SDK_PREFER_ORJSON", value) + if expected_result is not ValueError: + assert globus_sdk.config.get_prefer_orjson() == expected_result + + else: + with pytest.raises(expected_result): + globus_sdk.config.get_prefer_orjson() diff --git a/tests/unit/transport/test_transport_encoders.py b/tests/unit/transport/test_transport_encoders.py index 559227b9..f9b565ac 100644 --- a/tests/unit/transport/test_transport_encoders.py +++ b/tests/unit/transport/test_transport_encoders.py @@ -6,6 +6,13 @@ from globus_sdk._payload import GlobusPayload from globus_sdk.transport import FormRequestEncoder, JSONRequestEncoder, RequestEncoder +try: + import orjson # noqa: F401 + + has_orjson = True +except ImportError: + has_orjson = False + @pytest.mark.parametrize("data", ("foo", b"bar")) def test_text_request_encoder_accepts_string_data(data): @@ -159,3 +166,30 @@ def test_form_encoder_payload_preparation( headers={}, ) assert request.data == expected_data + + +@pytest.mark.skipif(has_orjson, reason="test requires that orjson is not installed") +@pytest.mark.parametrize("env_var_is_set", (True, False)) +def test_json_encoder_never_prefers_orjson_if_not_installed( + monkeypatch, env_var_is_set +): + if env_var_is_set: + monkeypatch.setenv("GLOBUS_SDK_PREFER_ORJSON", "true") + assert JSONRequestEncoder().use_orjson is False + + +@pytest.mark.skipif(not has_orjson, reason="test requires that orjson is installed") +@pytest.mark.parametrize("env_var_is_set", (True, False)) +def test_json_encoder_prefers_orjson_based_on_env_var(monkeypatch, env_var_is_set): + if env_var_is_set: + monkeypatch.setenv("GLOBUS_SDK_PREFER_ORJSON", "true") + + assert JSONRequestEncoder().use_orjson is env_var_is_set + + +@pytest.mark.parametrize("env_var_is_set", (True, False)) +@pytest.mark.parametrize("initarg", (True, False)) +def test_json_encoder_can_force_orjson_usage(monkeypatch, env_var_is_set, initarg): + if env_var_is_set: + monkeypatch.setenv("GLOBUS_SDK_PREFER_ORJSON", "true") + assert JSONRequestEncoder(use_orjson=initarg).use_orjson is initarg diff --git a/tox.ini b/tox.ini index c842d66e..3ec77025 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ envlist = test-lazy-imports coverage_clean py{3.14,3.13,3.12,3.11,3.10,3.9} + py{3.14,3.9}-orjson py3.9-mindeps py3.11-sphinxext coverage_report @@ -25,6 +26,9 @@ depends = coverage_clean,lint [testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-mindeps] deps = -r requirements/py{py_dot_ver}/test-mindeps.txt +[testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-orjson] +deps = -r requirements/py{py_dot_ver}/test-orjson.txt + [testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-sphinxext] deps = -r requirements/py{py_dot_ver}/test.txt @@ -112,6 +116,7 @@ deps = dependency-groups>=1,<2 commands = python -m dependency_groups test -o requirements/.test.in python -m dependency_groups typing -o requirements/.typing.in + python -m dependency_groups orjson -o requirements/.orjson.in python -m dependency_groups test-mindeps -o requirements/.test-mindeps.in python -m dependency_groups docs -o requirements/.docs.in [testenv:freezedeps-py{3.14,3.13,3.12,3.11,3.10,3.9}] @@ -127,6 +132,8 @@ commands = # Minimum dependencies are only tested against the lowest supported Python version. py3.9: pip-compile --strip-extras -q -U --resolver=backtracking .test-mindeps.in -o py{py_dot_ver}/test-mindeps.txt + # orjson testing happens on every version + pip-compile --strip-extras -q -U --resolver=backtracking -r .test.in -r .orjson.in -o py{py_dot_ver}/test-orjson.txt # The docs requirements are only generated for Python 3.11. py3.11: pip-compile --strip-extras -q -U --resolver=backtracking .docs.in -o py{py_dot_ver}/docs.txt