diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65692a3e..a61133da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: # and then finally an actual release version. actions/setup-python doesn't # support this for PyPy presently so we get no help there. # - # CPython -> 3.9.0-alpha - 3.9.X + # CPython -> 3.10.0-alpha - 3.10.X # PyPy -> pypy-3.7 # python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python.action), matrix.python.action))[startsWith(matrix.python.action, 'pypy')] }} # architecture: x64 @@ -82,12 +82,6 @@ jobs: - name: 🪟 runs-on: windows-latest python: - - name: CPython 3.8 - major_dot_minor: '3.8' - action: '3.8' - - name: CPython 3.9 - major_dot_minor: '3.9' - action: '3.9' - name: CPython 3.10 major_dot_minor: '3.10' action: '3.10' @@ -104,20 +98,6 @@ jobs: major_dot_minor: '3.14' action: '3.14' exclude: - - os: - name: "🍎" - runs-on: macos-15 - python: - name: "CPython 3.8" - major_dot_minor: '3.8' - action: '3.8' - - os: - name: "🍎" - runs-on: macos-15 - python: - name: "CPython 3.9" - major_dot_minor: '3.9' - action: '3.9' - os: name: "🍎" runs-on: macos-15 @@ -144,7 +124,7 @@ jobs: # and then finally an actual release version. actions/setup-python doesn't # support this for PyPy presently so we get no help there. # - # CPython -> 3.9.0-alpha - 3.9.X + # CPython -> 3.10.0-alpha - 3.10.X # PyPy -> pypy-3.7 python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python.action), matrix.python.action))[startsWith(matrix.python.action, 'pypy')] }} architecture: x64 @@ -160,100 +140,6 @@ jobs: ls -la - name: Run pytest - env: - BITSTRING_USE_RUST_CORE: ${{ matrix.rust.env_var }} - run: | - python -m pytest tests/ --benchmark-disable - - test-rust: - name: Test ${{ matrix.os.name }} ${{ matrix.python.name }} (Rust core) - if: always() - needs: - - test-standard - runs-on: ${{ matrix.os.runs-on }} - strategy: - fail-fast: false - matrix: - os: - - name: 🐧 - runs-on: ubuntu-latest - - name: 🍎 - runs-on: macos-15 - - name: 🪟 - runs-on: windows-latest - python: - - name: CPython 3.8 - major_dot_minor: '3.8' - action: '3.8' - - name: CPython 3.9 - major_dot_minor: '3.9' - action: '3.9' - - name: CPython 3.10 - major_dot_minor: '3.10' - action: '3.10' - - name: CPython 3.11 - major_dot_minor: '3.11' - action: '3.11' - - name: CPython 3.12 - major_dot_minor: '3.12' - action: '3.12' - - name: CPython 3.13 - major_dot_minor: '3.13' - action: '3.13' - - name: CPython 3.14 - major_dot_minor: '3.14' - action: '3.14' - exclude: - - os: - name: "🍎" - runs-on: macos-15 - python: - name: "CPython 3.8" - major_dot_minor: '3.8' - action: '3.8' - - os: - name: "🍎" - runs-on: macos-15 - python: - name: "CPython 3.9" - major_dot_minor: '3.9' - action: '3.9' - - os: - name: "🍎" - runs-on: macos-15 - python: - name: "CPython 3.10" - major_dot_minor: '3.10' - action: '3.10' - steps: - - uses: actions/checkout@v4 - with: - path: repo - - - name: Download package files - uses: actions/download-artifact@v4 - with: - name: packages - path: dist - - - uses: actions/setup-python@v5 - with: - python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python.action), matrix.python.action))[startsWith(matrix.python.action, 'pypy')] }} - architecture: x64 - - - name: Setup environment - run: | - python --version --version - # make sure we test the installed code - cp -R repo/tests/ tests/ - python -m pip install -r tests/requirements.txt - python -m pip install ./dist/*.whl - # show the directory contents for diagnostics - ls -la - - - name: Run pytest - env: - BITSTRING_USE_RUST_CORE: '1' run: | python -m pytest tests/ --benchmark-disable @@ -268,7 +154,6 @@ jobs: # a merge. - build - test-standard - - test-rust steps: - name: Require all successes uses: re-actors/alls-green@v1.2.2 diff --git a/.gitignore b/.gitignore index a05cefee..db884a7f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,9 @@ build dist manifest *egg-info -doc/_build \ No newline at end of file +doc/_build +/.hypothesis/ +/doc/html/ +/html/ +/tests/.hypothesis/ +/uv.lock diff --git a/README.md b/README.md index c1c68566..846ed971 100644 --- a/README.md +++ b/README.md @@ -16,18 +16,70 @@ ---- > [!NOTE] -> To see what been added, improved or fixed, and also to see what's coming in the next version, see the [release notes](https://github.com/scott-griffiths/bitstring/blob/main/release_notes.md). +> To see what's been added, improved or fixed, and also to see what's coming in the next version, see the [release notes](https://github.com/scott-griffiths/bitstring/blob/main/release_notes.md). -# News +> [!IMPORTANT] +> The `main` branch now contains the upcoming bitstring 5.0 work. It is visible here for testing and feedback, but it is not the current released version. The released PyPI line is still bitstring 4.x; use `pip install bitstring` or pin `bitstring<5` for the stable API. Source maintenance for 4.x is on the `4.x-maintenance` branch. -I'm rewriting the core of bitstring in Rust, and have released that cut-down and streamlined interface as the [tibs](https://github.com/scott-griffiths/tibs) package. It's now in beta, and feedback is welcome. Just `pip install tibs` to try it out. +# Version 5.0 preview + +Version 5.0 of bitstring is in development. This is a major update with breaking changes. + +Reasons to upgrade include: + +* Significantly better performance using the new [tibs](https://github.com/scott-griffiths/tibs) Rust core. +* A simpler model for sequential reading: `Bits` and `BitArray` store data, while `Reader` stores the current bit position. +* A clearer, more explicit API for construction, conversion and dtype names. +* Removal of old compatibility layers, including the `bitarray` dependency, LSB0 mode and the old stream classes. +* A smaller internal codebase that should be easier to maintain and optimise further. + +## Upgrading from version 4 + +If you have arrived here because bitstring 5 development work broke your code, the most practical fix may be to pin the latest version 4 release for now: + +```bash +pip install "bitstring<5" +``` + +Version 5 is worth moving to if you need the improved performance, want the cleaner current API, or want to keep up to +date with future bitstring development. The changes are quite broad, so if your existing code is stable and performance +is not a problem, there is no need to migrate immediately. + +For details on the source changes you may need, see the [Upgrading to version 5](https://bitstring.readthedocs.io/en/latest/upgrading_to_version_5.html) guide. + +## What's this 'tibs' thing that everyone is talking about? + +bitstring 5 is built on [tibs](https://github.com/scott-griffiths/tibs), a simpler and more focussed Python library for +binary data, written in Rust for speed. It's by the same author as bitstring so they can complement each other's needs. +If you want a leaner interface and do not need all of bitstring's higher-level format handling and historical API, tibs may be a better fit for new code. + +Tibs is still in beta but I would like it to get a wider range of users before hitting 1.0. Try it with: + +```bash +pip install tibs +``` + +Bitstring accepts tibs objects anywhere a bitstring-like value is expected: + +```python +from bitstring import Bits +from tibs import Mutibs, Tibs + +tibs_obj = Tibs.from_bin("1010") +bits = Bits(tibs_obj) # or Bits.from_tibs(tibs_obj) +same_data = bits.to_tibs() + +mutable = Mutibs.from_bin("111") +bits_copy = Bits.from_tibs(mutable) # Mutibs inputs are copied +``` [![tibs](https://raw.githubusercontent.com/scott-griffiths/tibs/main/doc/tibs.png)](https://github.com/scott-griffiths/tibs) A sleek Python library for your binary data +---- -# Overview +# Bitstring Overview * Efficiently store and manipulate binary data in idiomatic Python. * Create bitstrings from hex, octal, binary, files, formatted strings, bytes, integers and floats of different endiannesses. @@ -43,33 +95,43 @@ A sleek Python library for your binary data Extensive documentation for the bitstring library is available. Some starting points are given below: -* [Overview](https://bitstring.readthedocs.io/en/stable/index.html) -* [Quick Reference](https://bitstring.readthedocs.io/en/stable/quick_reference.html) -* [Full Reference](https://bitstring.readthedocs.io/en/stable/reference.html) +* [Released 4.x documentation](https://bitstring.readthedocs.io/en/stable/) +* [Upcoming 5.0 documentation](https://bitstring.readthedocs.io/en/latest/) +* [Upgrading from 4.x to 5.0](https://bitstring.readthedocs.io/en/latest/upgrading_to_version_5.html) -There is also an introductory walkthrough notebook on [binder](https://mybinder.org/v2/gh/scott-griffiths/bitstring/main?labpath=doc%2Fwalkthrough.ipynb). +There is also an introductory walkthrough notebook for the 5.0 branch on [binder](https://mybinder.org/v2/gh/scott-griffiths/bitstring/main?labpath=doc%2Fwalkthrough.ipynb). # Examples +These examples show the upcoming 5.0 API from `main`. + ### Installation -``` +Released 4.x version: + +```bash $ pip install bitstring ``` +5.0 preview from `main`: + +```bash +$ pip install "bitstring @ git+https://github.com/scott-griffiths/bitstring.git@main" +``` + ### Creation ```pycon ->>> from bitstring import Bits, BitArray, BitStream, pack +>>> from bitstring import Bits, BitArray, Reader, pack >>> a = BitArray(bin='00101') ->>> b = Bits(a_file_object) ->>> c = BitArray('0xff, 0b101, 0o65, uint6=22') ->>> d = pack('intle16, hex=a, 0b1', 100, a='0x34f') +>>> b = Bits.from_file(a_file_object) +>>> c = BitArray('0xff, 0b101, 0o65, u6=22') +>>> d = pack('ile16, hex=a, 0b1', 100, a='0x34f') >>> e = pack('<16h', *range(16)) ``` ### Different interpretations, slicing and concatenation ```pycon >>> a = BitArray('0x3348') ->>> a.hex, a.bin, a.uint, a.float, a.bytes +>>> a.hex, a.bin, a.u, a.f, a.bytes ('3348', '0011001101001000', 13128, 0.2275390625, b'3H') >>> a[10:3:-1].bin '0101100' @@ -79,13 +141,13 @@ BitArray('0x866906690669, 0b000') ### Reading data sequentially ```pycon ->>> b = BitStream('0x160120f') +>>> b = Reader(Bits('0x160120f')) >>> b.read(12).hex '160' >>> b.pos = 0 ->>> b.read('uint12') +>>> b.read('u12') 352 ->>> b.readlist('uint12, bin3') +>>> b.read_list('u12, bin3') [288, '111'] ``` @@ -93,7 +155,7 @@ BitArray('0x866906690669, 0b000') ```pycon >>> c = BitArray('0b00010010010010001111') # c.hex == '0x1248f' >>> c.find('0x48') -(8,) +8 >>> c.replace('0b001', '0xabc') >>> c.insert('0b0000', pos=3) >>> del c[12:16] @@ -102,12 +164,12 @@ BitArray('0x866906690669, 0b000') ### Arrays of fixed-length formats ```pycon >>> from bitstring import Array ->>> a = Array('uint7', [9, 100, 3, 1]) +>>> a = Array('u7', [9, 100, 3, 1]) >>> a.data BitArray('0x1390181') >>> a[::2] *= 5 >>> a -Array('uint7', [45, 100, 15, 1]) +Array('u7', [45, 100, 15, 1]) ``` diff --git a/bitstring/__init__.py b/bitstring/__init__.py index 438539b0..41505da8 100644 --- a/bitstring/__init__.py +++ b/bitstring/__init__.py @@ -7,14 +7,13 @@ Bits -- An immutable container for binary data. BitArray -- A mutable container for binary data. -ConstBitStream -- An immutable container with streaming methods. -BitStream -- A mutable container with streaming methods. +Reader -- Wraps a Bits or BitArray with a bit position for sequential reading. Array -- An efficient list-like container where each item has a fixed-length binary format. Dtype -- Encapsulate the data types used in the other classes. Functions: -pack -- Create a BitStream from a format string. +pack -- Create a Bits object from a format string. Data: @@ -55,37 +54,22 @@ THE SOFTWARE. """ -__version__ = "4.4.0" +__version__ = "5.0.0_beta1" __author__ = "Scott Griffiths" import sys -import os -import importlib - -# New ability to use tibs for core operations instead of bitarray. -# Tibs is written in Rust and is still in beta. Use the environment variable -# BITSTRING_USE_RUST_CORE=1 before importing the module to turn it on. -_env_core = os.getenv('BITSTRING_USE_RUST_CORE', '').strip().lower() -_USE_RUST_CORE = _env_core in ('1', 'true', 'yes', 'on') -if _USE_RUST_CORE: - bitstore = importlib.import_module('bitstring.bitstore_tibs') - bitstore_helpers = importlib.import_module('bitstring.bitstore_tibs_helpers') -else: - bitstore = importlib.import_module('bitstring.bitstore_bitarray') - bitstore_helpers = importlib.import_module('bitstring.bitstore_bitarray_helpers') -bitstore_common_helpers = importlib.import_module('bitstring.bitstore_common_helpers') +import bitstring.bitstore_tibs as bitstore # noqa: F401 - used as bitstring.bitstore by submodules. from .bits import Bits from .bitstring_options import Options from .bitarray_ import BitArray -from .bitstream import ConstBitStream, BitStream +from .reader import Reader from .methods import pack from .array_ import Array from .exceptions import Error, ReadError, InterpretError, ByteAlignError, CreationError from .dtypes import DtypeDefinition, dtype_register, Dtype -import types -from typing import List, Tuple, Literal +from typing import Literal from .mxfp import decompress_luts as mxfp_decompress_luts from .fp8 import decompress_luts as binary8_decompress_luts @@ -96,36 +80,6 @@ # The Options class returns a singleton. options = Options() -# These get defined properly by the module magic below. This just stops mypy complaining about them. -bytealigned = lsb0 = None - - -# An opaque way of adding module level properties. Taken from https://peps.python.org/pep-0549/ -# This is now deprecated. Use the options object directly instead. -class _MyModuleType(types.ModuleType): - @property - def bytealigned(self) -> bool: - """Determines whether a number of methods default to working only on byte boundaries.""" - return options.bytealigned - - @bytealigned.setter - def bytealigned(self, value: bool) -> None: - """Determines whether a number of methods default to working only on byte boundaries.""" - options.bytealigned = value - - @property - def lsb0(self) -> bool: - """If True, the least significant bit (the final bit) is indexed as bit zero.""" - return options.lsb0 - - @lsb0.setter - def lsb0(self, value: bool) -> None: - """If True, the least significant bit (the final bit) is indexed as bit zero.""" - options.lsb0 = value - - -sys.modules[__name__].__class__ = _MyModuleType - # These methods convert a bit length to the number of characters needed to print it for different interpretations. def hex_bits2chars(bitlength: int): @@ -155,7 +109,7 @@ def uint_bits2chars(bitlength: int): def int_bits2chars(bitlength: int): # How many characters is largest negative int of this length? (To include minus sign). - return len(str((-1 << (bitlength - 1)))) + return len(str(-1 << (bitlength - 1))) def float_bits2chars(bitlength: Literal[16, 32, 64]): @@ -215,7 +169,7 @@ def bfloat_bits2chars(_: Literal[16]): def bits_bits2chars(bitlength: int): # For bits type we can see how long it needs to be printed by trying any value - temp = Bits(bitlength) + temp = Bits.from_zeros(bitlength) return len(str(temp)) @@ -223,43 +177,78 @@ def bool_bits2chars(_: Literal[1]): # Bools are printed as 1 or 0, not True or False, so are one character each return 1 +byteorder: str = sys.byteorder + +native_setuint = Bits._setuintle if byteorder == 'little' else Bits._setuintbe +native_getuint = Bits._getuintle if byteorder == 'little' else Bits._getuintbe +native_readuint = Bits._readuintle if byteorder == 'little' else Bits._readuintbe +native_setint = Bits._setintle if byteorder == 'little' else Bits._setintbe +native_getint = Bits._getintle if byteorder == 'little' else Bits._getintbe +native_readint = Bits._readintle if byteorder == 'little' else Bits._readintbe +native_setfloat = Bits._setfloatle if byteorder == 'little' else Bits._setfloatbe +native_getfloat = Bits._getfloatle if byteorder == 'little' else Bits._getfloatbe +native_readfloat = Bits._readfloatle if byteorder == 'little' else Bits._readfloatbe dtype_definitions = [ # Integer types - DtypeDefinition('uint', Bits._setuint, Bits._getuint, int, False, uint_bits2chars, + DtypeDefinition('u', Bits._setuint, Bits._getuint, int, False, uint_bits2chars, + read_fn=Bits._readuint, description="a two's complement unsigned int"), - DtypeDefinition('uintle', Bits._setuintle, Bits._getuintle, int, False, uint_bits2chars, + DtypeDefinition('ule', Bits._setuintle, Bits._getuintle, int, False, uint_bits2chars, + read_fn=Bits._readuintle, allowed_lengths=(8, 16, 24, ...), description="a two's complement little-endian unsigned int"), - DtypeDefinition('uintbe', Bits._setuintbe, Bits._getuintbe, int, False, uint_bits2chars, + DtypeDefinition('ube', Bits._setuintbe, Bits._getuintbe, int, False, uint_bits2chars, + read_fn=Bits._readuintbe, allowed_lengths=(8, 16, 24, ...), description="a two's complement big-endian unsigned int"), - DtypeDefinition('int', Bits._setint, Bits._getint, int, True, int_bits2chars, + DtypeDefinition('une', native_setuint, native_getuint, int, False, uint_bits2chars, + read_fn=native_readuint, + allowed_lengths=(8, 16, 24, ...), description="a two's complement native-endian unsigned int"), + DtypeDefinition('i', Bits._setint, Bits._getint, int, True, int_bits2chars, + read_fn=Bits._readint, description="a two's complement signed int"), - DtypeDefinition('intle', Bits._setintle, Bits._getintle, int, True, int_bits2chars, + DtypeDefinition('ile', Bits._setintle, Bits._getintle, int, True, int_bits2chars, + read_fn=Bits._readintle, allowed_lengths=(8, 16, 24, ...), description="a two's complement little-endian signed int"), - DtypeDefinition('intbe', Bits._setintbe, Bits._getintbe, int, True, int_bits2chars, + DtypeDefinition('ibe', Bits._setintbe, Bits._getintbe, int, True, int_bits2chars, + read_fn=Bits._readintbe, allowed_lengths=(8, 16, 24, ...), description="a two's complement big-endian signed int"), + DtypeDefinition('ine', native_setint, native_getint, int, True, int_bits2chars, + read_fn=native_readint, + allowed_lengths=(8, 16, 24, ...), description="a two's complement native-endian signed int"), # String types DtypeDefinition('hex', Bits._sethex, Bits._gethex, str, False, hex_bits2chars, + read_fn=Bits._readhex, allowed_lengths=(0, 4, 8, ...), description="a hexadecimal string"), DtypeDefinition('bin', Bits._setbin, Bits._getbin, str, False, bin_bits2chars, + read_fn=Bits._readbin, description="a binary string"), DtypeDefinition('oct', Bits._setoct, Bits._getoct, str, False, oct_bits2chars, + read_fn=Bits._readoct, allowed_lengths=(0, 3, 6, ...), description="an octal string"), # Float types - DtypeDefinition('float', Bits._setfloatbe, Bits._getfloatbe, float, True, float_bits2chars, + DtypeDefinition('f', Bits._setfloatbe, Bits._getfloatbe, float, True, float_bits2chars, + read_fn=Bits._readfloatbe, allowed_lengths=(16, 32, 64), description="a big-endian floating point number"), - DtypeDefinition('floatle', Bits._setfloatle, Bits._getfloatle, float, True, float_bits2chars, + DtypeDefinition('fle', Bits._setfloatle, Bits._getfloatle, float, True, float_bits2chars, + read_fn=Bits._readfloatle, allowed_lengths=(16, 32, 64), description="a little-endian floating point number"), + DtypeDefinition('fne', native_setfloat, native_getfloat, float, True, float_bits2chars, + read_fn=native_readfloat, + allowed_lengths=(16, 32, 64), description="a native-endian floating point number"), DtypeDefinition('bfloat', Bits._setbfloatbe, Bits._getbfloatbe, float, True, bfloat_bits2chars, + read_fn=Bits._readbfloatbe, allowed_lengths=(16,), description="a 16 bit big-endian bfloat floating point number"), DtypeDefinition('bfloatle', Bits._setbfloatle, Bits._getbfloatle, float, True, bfloat_bits2chars, + read_fn=Bits._readbfloatle, allowed_lengths=(16,), description="a 16 bit little-endian bfloat floating point number"), # Other known length types DtypeDefinition('bits', Bits._setbits, Bits._getbits, Bits, False, bits_bits2chars, description="a bitstring object"), DtypeDefinition('bool', Bits._setbool, Bits._getbool, bool, False, bool_bits2chars, + read_fn=Bits._readbool, allowed_lengths=(1,), description="a bool (True or False)"), DtypeDefinition('bytes', Bits._setbytes, Bits._getbytes, bytes, False, bytes_bits2chars, + read_fn=Bits._readbytes, multiplier=8, description="a bytes object"), # Unknown length types DtypeDefinition('se', Bits._setse, Bits._getse, int, True, None, @@ -272,58 +261,75 @@ def bool_bits2chars(_: Literal[1]): variable_length=True, description="an unsigned interleaved exponential-Golomb code"), # Special case pad type DtypeDefinition('pad', Bits._setpad, Bits._getpad, None, False, None, + read_fn=Bits._readpad, description="a skipped section of padding"), # MXFP and IEEE 8-bit float types DtypeDefinition('p3binary', Bits._setp3binary, Bits._getp3binary, float, True, p3binary_bits2chars, + read_fn=Bits._readp3binary, allowed_lengths=(8,), description="an 8 bit float with binary8p3 format"), DtypeDefinition('p4binary', Bits._setp4binary, Bits._getp4binary, float, True, p4binary_bits2chars, + read_fn=Bits._readp4binary, allowed_lengths=(8,), description="an 8 bit float with binary8p4 format"), - DtypeDefinition('e4m3mxfp', Bits._sete4m3mxfp, Bits._gete4m3mxfp, float, True, e4m3mxfp_bits2chars, - allowed_lengths=(8,), description="an 8 bit float with MXFP E4M3 format"), - DtypeDefinition('e5m2mxfp', Bits._sete5m2mxfp, Bits._gete5m2mxfp, float, True, e5m2mxfp_bits2chars, - allowed_lengths=(8,), description="an 8 bit float with MXFP E5M2 format"), + DtypeDefinition('e4m3mxfp_saturate', Bits._sete4m3mxfp_saturate, Bits._gete4m3mxfp, float, True, e4m3mxfp_bits2chars, + read_fn=Bits._reade4m3mxfp, + allowed_lengths=(8,), description="an 8 bit float with MXFP E4M3 format, saturating out-of-range values"), + DtypeDefinition('e4m3mxfp_overflow', Bits._sete4m3mxfp_overflow, Bits._gete4m3mxfp, float, True, e4m3mxfp_bits2chars, + read_fn=Bits._reade4m3mxfp, + allowed_lengths=(8,), description="an 8 bit float with MXFP E4M3 format, overflowing out-of-range values"), + DtypeDefinition('e5m2mxfp_saturate', Bits._sete5m2mxfp_saturate, Bits._gete5m2mxfp, float, True, e5m2mxfp_bits2chars, + read_fn=Bits._reade5m2mxfp, + allowed_lengths=(8,), description="an 8 bit float with MXFP E5M2 format, saturating out-of-range values"), + DtypeDefinition('e5m2mxfp_overflow', Bits._sete5m2mxfp_overflow, Bits._gete5m2mxfp, float, True, e5m2mxfp_bits2chars, + read_fn=Bits._reade5m2mxfp, + allowed_lengths=(8,), description="an 8 bit float with MXFP E5M2 format, overflowing out-of-range values"), DtypeDefinition('e3m2mxfp', Bits._sete3m2mxfp, Bits._gete3m2mxfp, float, True, e3m2mxfp_bits2chars, + read_fn=Bits._reade3m2mxfp, allowed_lengths=(6,), description="a 6 bit float with MXFP E3M2 format"), DtypeDefinition('e2m3mxfp', Bits._sete2m3mxfp, Bits._gete2m3mxfp, float, True, e2m3mxfp_bits2chars, + read_fn=Bits._reade2m3mxfp, allowed_lengths=(6,), description="a 6 bit float with MXFP E2M3 format"), DtypeDefinition('e2m1mxfp', Bits._sete2m1mxfp, Bits._gete2m1mxfp, float, True, e2m1mxfp_bits2chars, + read_fn=Bits._reade2m1mxfp, allowed_lengths=(4,), description="a 4 bit float with MXFP E2M1 format"), DtypeDefinition('e8m0mxfp', Bits._sete8m0mxfp, Bits._gete8m0mxfp, float, False, e8m0mxfp_bits2chars, + read_fn=Bits._reade8m0mxfp, allowed_lengths=(8,), description="an 8 bit float with MXFP E8M0 format"), DtypeDefinition('mxint', Bits._setmxint, Bits._getmxint, float, True, mxint_bits2chars, + read_fn=Bits._readmxint, allowed_lengths=(8,), description="an 8 bit float with MXFP INT8 format"), ] -aliases: List[Tuple[str, str]] = [ +aliases: list[tuple[str, str]] = [ # Floats default to big endian - ('float', 'floatbe'), + ('f', 'float'), + ('f', 'floatbe'), + ('f', 'fbe'), ('bfloat', 'bfloatbe'), - # Some single letter aliases for popular types - ('int', 'i'), - ('uint', 'u'), - ('hex', 'h'), - ('oct', 'o'), - ('bin', 'b'), - ('float', 'f'), + # Longer compatibility aliases for the short core type names + ('i', 'int'), + ('u', 'uint'), + + # Longer compatibility aliases for endian-specific type names + ('ube', 'uintbe'), + ('ule', 'uintle'), + ('une', 'uintne'), + ('ibe', 'intbe'), + ('ile', 'intle'), + ('ine', 'intne'), + ('fle', 'floatle'), + ('fne', 'floatne'), ] # Create native-endian aliases depending on the byteorder of the system -byteorder: str = sys.byteorder if byteorder == 'little': aliases.extend([ - ('uintle', 'uintne'), - ('intle', 'intne'), - ('floatle', 'floatne'), ('bfloatle', 'bfloatne'), ]) else: aliases.extend([ - ('uintbe', 'uintne'), - ('intbe', 'intne'), - ('floatbe', 'floatne'), ('bfloatbe', 'bfloatne'), ]) @@ -341,12 +347,6 @@ def bool_bits2chars(_: Literal[1]): Bits.__doc__ = Bits.__doc__.replace('[GENERATED_PROPERTY_DESCRIPTIONS]', property_docstring) if BitArray.__doc__ is not None: BitArray.__doc__ = BitArray.__doc__.replace('[GENERATED_PROPERTY_DESCRIPTIONS]', property_docstring) -if ConstBitStream.__doc__ is not None: - ConstBitStream.__doc__ = ConstBitStream.__doc__.replace('[GENERATED_PROPERTY_DESCRIPTIONS]', property_docstring) -if BitStream.__doc__ is not None: - BitStream.__doc__ = BitStream.__doc__.replace('[GENERATED_PROPERTY_DESCRIPTIONS]', property_docstring) - - -__all__ = ['ConstBitStream', 'BitStream', 'BitArray', 'Array', +__all__ = ['Reader', 'BitArray', 'Array', 'Bits', 'pack', 'Error', 'ReadError', 'InterpretError', - 'ByteAlignError', 'CreationError', 'bytealigned', 'lsb0', 'Dtype', 'options'] + 'ByteAlignError', 'CreationError', 'Dtype', 'options'] diff --git a/bitstring/__main__.py b/bitstring/__main__.py deleted file mode 100644 index f5949ff0..00000000 --- a/bitstring/__main__.py +++ /dev/null @@ -1,50 +0,0 @@ -import sys -from bitstring.bits import Bits -from bitstring.dtypes import Register - -dtype_register = Register() - - -def main() -> None: - # check if final parameter is an interpretation string - fp = sys.argv[-1] - if fp in ['-h', '--help'] or len(sys.argv) == 1: - print("""Create and interpret a bitstring from command-line parameters. - -Command-line parameters are concatenated and a bitstring created -from them. If the final parameter is either an interpretation string -or ends with a '.' followed by an interpretation string then that -interpretation of the bitstring will be used when printing it. - -Typical usage might be invoking the Python module from a console -as a one-off calculation: - -$ python -m bitstring int:16=-400 -0xfe70 -$ python -m bitstring float:32=0.2 bin -00111110010011001100110011001101 -$ python -m bitstring 0xff 3*0b01,0b11 uint -65367 -$ python -m bitstring hex=01, uint:12=352.hex -01160 - """) - return - if fp in dtype_register.names: - # concatenate all other parameters and interpret using the final one - b1 = Bits(','.join(sys.argv[1: -1])) - print(b1._readtoken(fp, 0, b1.__len__())[0]) - else: - # does final parameter end with a dot then an interpretation string? - interp = fp[fp.rfind('.') + 1:] - if interp in dtype_register.names: - sys.argv[-1] = fp[:fp.rfind('.')] - b1 = Bits(','.join(sys.argv[1:])) - print(b1._readtoken(interp, 0, b1.__len__())[0]) - else: - # No interpretation - just use default print - b1 = Bits(','.join(sys.argv[1:])) - print(b1) - - -if __name__ == '__main__': - main() diff --git a/bitstring/array_.py b/bitstring/array_.py index fe75bc51..f0c210f9 100644 --- a/bitstring/array_.py +++ b/bitstring/array_.py @@ -4,7 +4,8 @@ import numbers from collections.abc import Sized from bitstring.exceptions import CreationError -from typing import Union, List, Iterable, Any, Optional, BinaryIO, overload, TextIO +from typing import Any, BinaryIO, overload, TextIO +from collections.abc import Iterable from bitstring.bits import Bits, BitsType from bitstring.bitarray_ import BitArray from bitstring.dtypes import Dtype, dtype_register @@ -15,11 +16,13 @@ import operator import io import sys +import bitstring # The possible types stored in each element of the Array -ElementType = Union[float, str, int, bytes, bool, Bits] +ElementType = float | str | int | bytes | bool | Bits options = Options() +MutableBitStore = bitstring.bitstore.MutableBitStore class Array: @@ -28,7 +31,7 @@ class Array: format. a = Array('>H', [1, 15, 105]) - b = Array('int5', [-9, 0, 4]) + b = Array('i5', [-9, 0, 4]) The Array data is stored compactly as a BitArray object and the Array behaves very like a list of items of the given format. Both the Array data and fmt properties can be freely @@ -42,14 +45,14 @@ class Array: byteswap() -- Change byte endianness of all items. count() -- Count the number of occurences of a value. extend() -- Append new items to the end of the Array from an iterable. - fromfile() -- Append items read from a file object. + from_file() -- Append items read from a file object. insert() -- Insert an item at a given position. pop() -- Remove and return an item. pp() -- Pretty print the Array. reverse() -- Reverse the order of all items. - tobytes() -- Return Array data as bytes object, padding with zero bits at the end if needed. - tofile() -- Write Array data to a file, padding with zero bits at the end if needed. - tolist() -- Return Array items as a list. + to_bytes() -- Return Array data as bytes object, padding with zero bits at the end if needed. + to_file() -- Write Array data to a file, padding with zero bits at the end if needed. + to_list() -- Return Array items as a list. Special methods: @@ -67,8 +70,8 @@ class Array: """ - def __init__(self, dtype: Union[str, Dtype], initializer: Optional[Union[int, Array, array.array, Iterable, Bits, bytes, bytearray, memoryview, BinaryIO]] = None, - trailing_bits: Optional[BitsType] = None) -> None: + def __init__(self, dtype: str | Dtype, initializer: int | Array | array.array | Iterable | Bits | bytes | bytearray | memoryview | BinaryIO | None = None, + trailing_bits: BitsType | None = None) -> None: self.data = BitArray() if isinstance(dtype, Dtype) and dtype.scale == 'auto': if isinstance(initializer, (int, Bits, bytes, bytearray, memoryview, BinaryIO)): @@ -81,11 +84,11 @@ def __init__(self, dtype: Union[str, Dtype], initializer: Optional[Union[int, Ar raise CreationError(e) if isinstance(initializer, numbers.Integral): - self.data = BitArray(initializer * self._dtype.bitlength) + self.data = BitArray.from_zeros(initializer * self._dtype.bitlength) elif isinstance(initializer, (Bits, bytes, bytearray, memoryview)): self.data += initializer elif isinstance(initializer, io.BufferedReader): - self.fromfile(initializer) + self.from_file(initializer) elif initializer is not None: self.extend(initializer) @@ -95,7 +98,7 @@ def __init__(self, dtype: Union[str, Dtype], initializer: Optional[Union[int, Ar _largest_values = None @staticmethod - def _calculate_auto_scale(initializer, name: str, length: Optional[int]) -> float: + def _calculate_auto_scale(initializer, name: str, length: int | None) -> float: # Now need to find the largest power of 2 representable with this format. if Array._largest_values is None: Array._largest_values = { @@ -103,16 +106,18 @@ def _calculate_auto_scale(initializer, name: str, length: Optional[int]) -> floa 'e2m1mxfp4': Bits('0b0111').e2m1mxfp4, # 6.0 'e2m3mxfp6': Bits('0b011111').e2m3mxfp6, # 7.5 'e3m2mxfp6': Bits('0b011111').e3m2mxfp6, # 28.0 - 'e4m3mxfp8': Bits('0b01111110').e4m3mxfp8, # 448.0 - 'e5m2mxfp8': Bits('0b01111011').e5m2mxfp8, # 57344.0 + 'e4m3mxfp_saturate8': Bits('0b01111110').e4m3mxfp_saturate8, # 448.0 + 'e4m3mxfp_overflow8': Bits('0b01111110').e4m3mxfp_overflow8, # 448.0 + 'e5m2mxfp_saturate8': Bits('0b01111011').e5m2mxfp_saturate8, # 57344.0 + 'e5m2mxfp_overflow8': Bits('0b01111011').e5m2mxfp_overflow8, # 57344.0 'p4binary8': Bits('0b01111110').p4binary8, # 224.0 'p3binary8': Bits('0b01111110').p3binary8, # 49152.0 - 'float16': Bits('0x7bff').float16, # 65504.0 + 'f16': Bits('0x7bff').f16, # 65504.0 # The bfloat range is so large the scaling algorithm doesn't work well, so I'm disallowing it. # 'bfloat16': Bits('0x7f7f').bfloat16, # 3.38953139e38, } if f'{name}{length}' in Array._largest_values.keys(): - float_values = Array('float64', initializer).tolist() + float_values = Array('f64', initializer).to_list() if not float_values: raise ValueError("Can't calculate an 'auto' scale with an empty Array initializer.") max_float_value = max(abs(x) for x in float_values) @@ -135,7 +140,10 @@ def _calculate_auto_scale(initializer, name: str, length: Optional[int]) -> floa @property def itemsize(self) -> int: - return self._dtype.length + bitlength = self._dtype.bitlength + if bitlength is None: + raise ValueError("A fixed length format is needed for an Array.") + return bitlength @property def trailing_bits(self) -> BitArray: @@ -147,10 +155,10 @@ def dtype(self) -> Dtype: return self._dtype @dtype.setter - def dtype(self, new_dtype: Union[str, Dtype]) -> None: + def dtype(self, new_dtype: str | Dtype) -> None: self._set_dtype(new_dtype) - def _set_dtype(self, new_dtype: Union[str, Dtype]) -> None: + def _set_dtype(self, new_dtype: str | Dtype) -> None: if isinstance(new_dtype, Dtype): self._dtype = new_dtype else: @@ -170,13 +178,13 @@ def _set_dtype(self, new_dtype: Union[str, Dtype]) -> None: def _create_element(self, value: ElementType) -> Bits: """Create Bits from value according to the token_name and token_length""" - b = self._dtype.build(value) - if len(b) != self._dtype.length: + b = self._dtype.pack(value) + if len(b) != self.itemsize: raise ValueError(f"The value {value!r} has the wrong length for the format '{self._dtype}'.") return b def __len__(self) -> int: - return len(self.data) // self._dtype.length + return len(self.data) // self.itemsize @overload def __getitem__(self, key: slice) -> Array: @@ -186,26 +194,28 @@ def __getitem__(self, key: slice) -> Array: def __getitem__(self, key: int) -> ElementType: ... - def __getitem__(self, key: Union[slice, int]) -> Union[Array, ElementType]: + def __getitem__(self, key: slice | int) -> Array | ElementType: if isinstance(key, slice): start, stop, step = key.indices(len(self)) if step != 1: d = BitArray() - for s in range(start * self._dtype.length, stop * self._dtype.length, step * self._dtype.length): - d.append(self.data[s: s + self._dtype.length]) + itemsize = self.itemsize + for s in range(start * itemsize, stop * itemsize, step * itemsize): + d.append(self.data[s: s + itemsize]) a = self.__class__(self._dtype) a.data = d return a else: + itemsize = self.itemsize a = self.__class__(self._dtype) - a.data = self.data[start * self._dtype.length: stop * self._dtype.length] + a.data = self.data[start * itemsize: stop * itemsize] return a else: if key < 0: key += len(self) if key < 0 or key >= len(self): raise IndexError(f"Index {key} out of range for Array of length {len(self)}.") - return self._dtype.read_fn(self.data, start=self._dtype.length * key) + return self._dtype.read_fn(self.data, start=self.itemsize * key) @overload def __setitem__(self, key: slice, value: Iterable[ElementType]) -> None: @@ -215,7 +225,7 @@ def __setitem__(self, key: slice, value: Iterable[ElementType]) -> None: def __setitem__(self, key: int, value: ElementType) -> None: ... - def __setitem__(self, key: Union[slice, int], value: Union[Iterable[ElementType], ElementType]) -> None: + def __setitem__(self, key: slice | int, value: Iterable[ElementType] | ElementType) -> None: if isinstance(key, slice): start, stop, step = key.indices(len(self)) if not isinstance(value, Iterable): @@ -224,14 +234,16 @@ def __setitem__(self, key: Union[slice, int], value: Union[Iterable[ElementType] new_data = BitArray() for x in value: new_data += self._create_element(x) - self.data[start * self._dtype.length: stop * self._dtype.length] = new_data + itemsize = self.itemsize + self.data[start * itemsize: stop * itemsize] = new_data return items_in_slice = len(range(start, stop, step)) if not isinstance(value, Sized): value = list(value) if len(value) == items_in_slice: + itemsize = self.itemsize for s, v in zip(range(start, stop, step), value): - self.data.overwrite(self._create_element(v), s * self._dtype.length) + self.data.overwrite(self._create_element(v), s * itemsize) else: raise ValueError(f"Can't assign {len(value)} values to an extended slice of length {items_in_slice}.") else: @@ -239,52 +251,61 @@ def __setitem__(self, key: Union[slice, int], value: Union[Iterable[ElementType] key += len(self) if key < 0 or key >= len(self): raise IndexError(f"Index {key} out of range for Array of length {len(self)}.") - start = self._dtype.length * key + start = self.itemsize * key self.data.overwrite(self._create_element(value), start) return - def __delitem__(self, key: Union[slice, int]) -> None: + def __delitem__(self, key: slice | int) -> None: if isinstance(key, slice): start, stop, step = key.indices(len(self)) if step == 1: - self.data.__delitem__(slice(start * self._dtype.length, stop * self._dtype.length)) + itemsize = self.itemsize + self.data.__delitem__(slice(start * itemsize, stop * itemsize)) return # We need to delete from the end or the earlier positions will change r = reversed(range(start, stop, step)) if step > 0 else range(start, stop, step) + itemsize = self.itemsize for s in r: - self.data.__delitem__(slice(s * self._dtype.length, (s + 1) * self._dtype.length)) + self.data.__delitem__(slice(s * itemsize, (s + 1) * itemsize)) else: if key < 0: key += len(self) if key < 0 or key >= len(self): raise IndexError - start = self._dtype.length * key - del self.data[start: start + self._dtype.length] + itemsize = self.itemsize + start = itemsize * key + del self.data[start: start + itemsize] def __repr__(self) -> str: - list_str = f"{self.tolist()}" - trailing_bit_length = len(self.data) % self._dtype.length + list_str = f"{self.to_list()}" + trailing_bit_length = len(self.data) % self.itemsize final_str = "" if trailing_bit_length == 0 else ", trailing_bits=" + repr( self.data[-trailing_bit_length:]) return f"Array('{self._dtype}', {list_str}{final_str})" - def astype(self, dtype: Union[str, Dtype]) -> Array: + def astype(self, dtype: str | Dtype) -> Array: """Return Array with elements of new dtype, initialised from current Array.""" - new_array = self.__class__(dtype, self.tolist()) + new_array = self.__class__(dtype, self.to_list()) return new_array - def tolist(self) -> List[ElementType]: + def to_list(self) -> list[ElementType]: + itemsize = self.itemsize return [self._dtype.read_fn(self.data, start=start) - for start in range(0, len(self.data) - self._dtype.length + 1, self._dtype.length)] + for start in range(0, len(self.data) - itemsize + 1, itemsize)] + + def tolist(self) -> list[ElementType]: + """Compatibility alias for :meth:`to_list`.""" + return self.to_list() def append(self, x: ElementType) -> None: - if len(self.data) % self._dtype.length != 0: + if len(self.data) % self.itemsize != 0: raise ValueError("Cannot append to Array as its length is not a multiple of the format length.") self.data += self._create_element(x) - def extend(self, iterable: Union[Array, array.array, Iterable[Any]]) -> None: - if len(self.data) % self._dtype.length != 0: - raise ValueError(f"Cannot extend Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({self._dtype.length} bits).") + def extend(self, iterable: Array | array.array | Iterable[Any]) -> None: + itemsize = self.itemsize + if len(self.data) % itemsize != 0: + raise ValueError(f"Cannot extend Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({itemsize} bits).") if isinstance(iterable, Array): if self._dtype.name != iterable._dtype.name or self._dtype.length != iterable._dtype.length: raise TypeError( @@ -297,7 +318,7 @@ def extend(self, iterable: Union[Array, array.array, Iterable[Any]]) -> None: if name_value is None: raise ValueError(f"Cannot extend from array with typecode {iterable.typecode}.") other_dtype = dtype_register.get_dtype(*name_value, scale=None) - if self._dtype.name != other_dtype.name or self._dtype.length != other_dtype.length: + if self._dtype.name != other_dtype.name or self.itemsize != other_dtype.bitlength: raise ValueError( f"Cannot extend an Array with format '{self._dtype}' from an array with typecode '{iterable.typecode}'.") self.data += iterable.tobytes() @@ -311,8 +332,9 @@ def insert(self, i: int, x: ElementType) -> None: """Insert a new element into the Array at position i. """ - i = min(i, len(self)) # Inserting beyond len of array inserts at the end (copying standard behaviour) - self.data.insert(self._create_element(x), i * self._dtype.length) + # Match list.insert semantics: clamp both high and low indices. + i = max(min(i, len(self)), -len(self)) + self.data.insert(self._create_element(x), i * self.itemsize) def pop(self, i: int = -1) -> ElementType: """Return and remove an element of the Array. @@ -332,9 +354,9 @@ def byteswap(self) -> None: If the Array format is not a whole number of bytes a ValueError will be raised. """ - if self._dtype.length % 8 != 0: + if self.itemsize % 8 != 0: raise ValueError( - f"byteswap can only be used for whole-byte elements. The '{self._dtype}' format is {self._dtype.length} bits long.") + f"byteswap can only be used for whole-byte elements. The '{self._dtype}' format is {self.itemsize} bits long.") self.data.byteswap(self.itemsize // 8) def count(self, value: ElementType) -> int: @@ -345,51 +367,79 @@ def count(self, value: ElementType) -> int: For floating point types using a value of float('nan') will count the number of elements that are NaN. """ - if math.isnan(value): - return sum(math.isnan(i) for i in self) + def is_nan(x: Any) -> bool: + try: + return math.isnan(x) + except TypeError: + return False + + if is_nan(value): + return sum(is_nan(i) for i in self) else: return sum(i == value for i in self) - def tobytes(self) -> bytes: + def to_bytes(self) -> bytes: """Return the Array data as a bytes object, padding with zero bits if needed. Up to seven zero bits will be added at the end to byte align. """ - return self.data.tobytes() + return self.data.to_bytes() - def tofile(self, f: BinaryIO) -> None: + def tobytes(self) -> bytes: + """Compatibility alias for :meth:`to_bytes`.""" + return self.to_bytes() + + def to_file(self, f: BinaryIO) -> None: """Write the Array data to a file object, padding with zero bits if needed. Up to seven zero bits will be added at the end to byte align. """ - self.data.tofile(f) + self.data.to_file(f) - def fromfile(self, f: BinaryIO, n: Optional[int] = None) -> None: + def tofile(self, f: BinaryIO) -> None: + """Compatibility alias for :meth:`to_file`.""" + self.to_file(f) + + def from_file(self, f: BinaryIO, n: int | None = None) -> None: trailing_bit_length = len(self.data) % self._dtype.bitlength if trailing_bit_length != 0: raise ValueError(f"Cannot extend Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({self._dtype.bitlength} bits).") + if n is not None and n < 0: + raise ValueError("n must be >= 0.") - new_data = Bits(f) - max_items = len(new_data) // self._dtype.length + item_bits = self.itemsize + if n is None: + b = f.read() + else: + bytes_needed = (n * item_bits + 7) // 8 + b = f.read(bytes_needed) + max_items = len(b) * 8 // item_bits items_to_append = max_items if n is None else min(n, max_items) - self.data += new_data[0: items_to_append * self._dtype.bitlength] + bits_to_append = items_to_append * item_bits + if bits_to_append: + self.data._bitstore += MutableBitStore.from_bytes(b, length=bits_to_append) if n is not None and items_to_append < n: raise EOFError(f"Only {items_to_append} were appended, not the {n} items requested.") + def fromfile(self, f: BinaryIO, n: int | None = None) -> None: + """Compatibility alias for :meth:`from_file`.""" + self.from_file(f, n) + def reverse(self) -> None: - trailing_bit_length = len(self.data) % self._dtype.length + itemsize = self.itemsize + trailing_bit_length = len(self.data) % itemsize if trailing_bit_length != 0: - raise ValueError(f"Cannot reverse the items in the Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({self._dtype.length} bits).") - for start_bit in range(0, len(self.data) // 2, self._dtype.length): - start_swap_bit = len(self.data) - start_bit - self._dtype.length - temp = self.data[start_bit: start_bit + self._dtype.length] - self.data[start_bit: start_bit + self._dtype.length] = self.data[ - start_swap_bit: start_swap_bit + self._dtype.length] - self.data[start_swap_bit: start_swap_bit + self._dtype.length] = temp - - def pp(self, fmt: Optional[str] = None, width: int = 120, + raise ValueError(f"Cannot reverse the items in the Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({itemsize} bits).") + for start_bit in range(0, len(self.data) // 2, itemsize): + start_swap_bit = len(self.data) - start_bit - itemsize + temp = self.data[start_bit: start_bit + itemsize] + self.data[start_bit: start_bit + itemsize] = self.data[ + start_swap_bit: start_swap_bit + itemsize] + self.data[start_swap_bit: start_swap_bit + itemsize] = temp + + def pp(self, fmt: str | None = None, width: int = 120, show_offset: bool = True, stream: TextIO = sys.stdout) -> None: """Pretty-print the Array contents. @@ -439,7 +489,7 @@ def pp(self, fmt: Optional[str] = None, width: int = 120, length = len(self.data) // token_length len_str = colour.green + str(length) + colour.off stream.write(f"<{self.__class__.__name__} {tidy_fmt}, length={len_str}, itemsize={token_length} bits, total data size={(len(self.data) + 7) // 8} bytes> [\n") - data._pp(dtype1, dtype2, token_length, width, sep, format_sep, show_offset, stream, False, token_length) + data._pp(dtype1, dtype2, token_length, width, sep, format_sep, show_offset, stream, token_length) stream.write("]") if trailing_bit_length != 0: stream.write(" + trailing_bits = " + str(self.data[-trailing_bit_length:])) @@ -464,23 +514,24 @@ def equals(self, other: Any) -> bool: return False if len(self) != len(other): return False - if self.tolist() != other.tolist(): + if self.to_list() != other.tolist(): return False return True return False def __iter__(self) -> Iterable[ElementType]: start = 0 + itemsize = self.itemsize for _ in range(len(self)): yield self._dtype.read_fn(self.data, start=start) - start += self._dtype.length + start += itemsize def __copy__(self) -> Array: a_copy = self.__class__(self._dtype) a_copy.data = copy.copy(self.data) return a_copy - def _apply_op_to_all_elements(self, op, value: Union[int, float, None], is_comparison: bool = False) -> Array: + def _apply_op_to_all_elements(self, op, value: int | float | None, is_comparison: bool = False) -> Array: """Apply op with value to each element of the Array and return a new Array""" new_array = self.__class__('bool' if is_comparison else self._dtype) new_data = BitArray() @@ -492,8 +543,9 @@ def partial_op(a): else: def partial_op(a): return op(a) + itemsize = self.itemsize for i in range(len(self)): - v = self._dtype.read_fn(self.data, start=self._dtype.length * i) + v = self._dtype.read_fn(self.data, start=itemsize * i) try: new_data.append(new_array._create_element(partial_op(v))) except (CreationError, ZeroDivisionError, ValueError) as e: @@ -507,14 +559,15 @@ def partial_op(a): new_array.data = new_data return new_array - def _apply_op_to_all_elements_inplace(self, op, value: Union[int, float]) -> Array: + def _apply_op_to_all_elements_inplace(self, op, value: int | float) -> Array: """Apply op with value to each element of the Array in place.""" # This isn't really being done in-place, but it's simpler and faster for now? new_data = BitArray() failures = index = 0 msg = '' + itemsize = self.itemsize for i in range(len(self)): - v = self._dtype.read_fn(self.data, start=self._dtype.length * i) + v = self._dtype.read_fn(self.data, start=itemsize * i) try: new_data.append(self._create_element(op(v, value))) except (CreationError, ZeroDivisionError, ValueError) as e: @@ -537,10 +590,11 @@ def _apply_bitwise_op_to_all_elements(self, op, value: BitsType) -> Array: def _apply_bitwise_op_to_all_elements_inplace(self, op, value: BitsType) -> Array: """Apply op with value to each element of the Array as an unsigned integer in place.""" value = BitArray._create_from_bitstype(value) - if len(value) != self._dtype.length: - raise ValueError(f"Bitwise op needs a bitstring of length {self._dtype.length} to match format {self._dtype}.") - for start in range(0, len(self) * self._dtype.length, self._dtype.length): - self.data[start: start + self._dtype.length] = op(self.data[start: start + self._dtype.length], value) + itemsize = self.itemsize + if len(value) != itemsize: + raise ValueError(f"Bitwise op needs a bitstring of length {itemsize} to match format {self._dtype}.") + for start in range(0, len(self) * itemsize, itemsize): + self.data[start: start + itemsize] = op(self.data[start: start + itemsize], value) return self def _apply_op_between_arrays(self, op, other: Array, is_comparison: bool = False) -> Array: @@ -559,9 +613,11 @@ def _apply_op_between_arrays(self, op, other: Array, is_comparison: bool = False new_data = BitArray() failures = index = 0 msg = '' + itemsize = self.itemsize + other_itemsize = other.itemsize for i in range(len(self)): - a = self._dtype.read_fn(self.data, start=self._dtype.length * i) - b = other._dtype.read_fn(other.data, start=other._dtype.length * i) + a = self._dtype.read_fn(self.data, start=itemsize * i) + b = other._dtype.read_fn(other.data, start=other_itemsize * i) try: new_data.append(new_array._create_element(op(a, b))) except (CreationError, ValueError, ZeroDivisionError) as e: @@ -575,6 +631,13 @@ def _apply_op_between_arrays(self, op, other: Array, is_comparison: bool = False new_array.data = new_data return new_array + def _apply_op_between_arrays_inplace(self, op, other: Array) -> Array: + """Apply op between Arrays and update self in place.""" + result = self._apply_op_between_arrays(op, other) + self._dtype = result._dtype + self.data = result.data + return self + @classmethod def _promotetype(cls, type1: Dtype, type2: Dtype) -> Dtype: """When combining types which one wins? @@ -610,85 +673,85 @@ def is_int(x): return x.return_type is int or x.return_type is bool # Operators between Arrays or an Array and scalar value - def __add__(self, other: Union[int, float, Array]) -> Array: + def __add__(self, other: int | float | Array) -> Array: """Add int or float to all elements.""" if isinstance(other, Array): return self._apply_op_between_arrays(operator.add, other) return self._apply_op_to_all_elements(operator.add, other) - def __iadd__(self, other: Union[int, float, Array]) -> Array: + def __iadd__(self, other: int | float | Array) -> Array: if isinstance(other, Array): - return self._apply_op_between_arrays(operator.add, other) + return self._apply_op_between_arrays_inplace(operator.add, other) return self._apply_op_to_all_elements_inplace(operator.add, other) - def __isub__(self, other: Union[int, float, Array]) -> Array: + def __isub__(self, other: int | float | Array) -> Array: if isinstance(other, Array): - return self._apply_op_between_arrays(operator.sub, other) + return self._apply_op_between_arrays_inplace(operator.sub, other) return self._apply_op_to_all_elements_inplace(operator.sub, other) - def __sub__(self, other: Union[int, float, Array]) -> Array: + def __sub__(self, other: int | float | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.sub, other) return self._apply_op_to_all_elements(operator.sub, other) - def __mul__(self, other: Union[int, float, Array]) -> Array: + def __mul__(self, other: int | float | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.mul, other) return self._apply_op_to_all_elements(operator.mul, other) - def __imul__(self, other: Union[int, float, Array]) -> Array: + def __imul__(self, other: int | float | Array) -> Array: if isinstance(other, Array): - return self._apply_op_between_arrays(operator.mul, other) + return self._apply_op_between_arrays_inplace(operator.mul, other) return self._apply_op_to_all_elements_inplace(operator.mul, other) - def __floordiv__(self, other: Union[int, float, Array]) -> Array: + def __floordiv__(self, other: int | float | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.floordiv, other) return self._apply_op_to_all_elements(operator.floordiv, other) - def __ifloordiv__(self, other: Union[int, float, Array]) -> Array: + def __ifloordiv__(self, other: int | float | Array) -> Array: if isinstance(other, Array): - return self._apply_op_between_arrays(operator.floordiv, other) + return self._apply_op_between_arrays_inplace(operator.floordiv, other) return self._apply_op_to_all_elements_inplace(operator.floordiv, other) - def __truediv__(self, other: Union[int, float, Array]) -> Array: + def __truediv__(self, other: int | float | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.truediv, other) return self._apply_op_to_all_elements(operator.truediv, other) - def __itruediv__(self, other: Union[int, float, Array]) -> Array: + def __itruediv__(self, other: int | float | Array) -> Array: if isinstance(other, Array): - return self._apply_op_between_arrays(operator.truediv, other) + return self._apply_op_between_arrays_inplace(operator.truediv, other) return self._apply_op_to_all_elements_inplace(operator.truediv, other) - def __rshift__(self, other: Union[int, Array]) -> Array: + def __rshift__(self, other: int | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.rshift, other) return self._apply_op_to_all_elements(operator.rshift, other) - def __lshift__(self, other: Union[int, Array]) -> Array: + def __lshift__(self, other: int | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.lshift, other) return self._apply_op_to_all_elements(operator.lshift, other) - def __irshift__(self, other: Union[int, Array]) -> Array: + def __irshift__(self, other: int | Array) -> Array: if isinstance(other, Array): - return self._apply_op_between_arrays(operator.rshift, other) + return self._apply_op_between_arrays_inplace(operator.rshift, other) return self._apply_op_to_all_elements_inplace(operator.rshift, other) - def __ilshift__(self, other: Union[int, Array]) -> Array: + def __ilshift__(self, other: int | Array) -> Array: if isinstance(other, Array): - return self._apply_op_between_arrays(operator.lshift, other) + return self._apply_op_between_arrays_inplace(operator.lshift, other) return self._apply_op_to_all_elements_inplace(operator.lshift, other) - def __mod__(self, other: Union[int, Array]) -> Array: + def __mod__(self, other: int | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.mod, other) return self._apply_op_to_all_elements(operator.mod, other) - def __imod__(self, other: Union[int, Array]) -> Array: + def __imod__(self, other: int | Array) -> Array: if isinstance(other, Array): - return self._apply_op_between_arrays(operator.mod, other) + return self._apply_op_between_arrays_inplace(operator.mod, other) return self._apply_op_to_all_elements_inplace(operator.mod, other) # Bitwise operators @@ -713,13 +776,13 @@ def __ixor__(self, other: BitsType) -> Array: # Reverse operators between a scalar value and an Array - def __rmul__(self, other: Union[int, float]) -> Array: + def __rmul__(self, other: int | float) -> Array: return self._apply_op_to_all_elements(operator.mul, other) - def __radd__(self, other: Union[int, float]) -> Array: + def __radd__(self, other: int | float) -> Array: return self._apply_op_to_all_elements(operator.add, other) - def __rsub__(self, other: Union[int, float]) -> Array: + def __rsub__(self, other: int | float) -> Array: # i - A == (-A) + i neg = self._apply_op_to_all_elements(operator.neg, None) return neg._apply_op_to_all_elements(operator.add, other) @@ -737,22 +800,22 @@ def __rxor__(self, other: BitsType) -> Array: # Comparison operators - def __lt__(self, other: Union[int, float, Array]) -> Array: + def __lt__(self, other: int | float | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.lt, other, is_comparison=True) return self._apply_op_to_all_elements(operator.lt, other, is_comparison=True) - def __gt__(self, other: Union[int, float, Array]) -> Array: + def __gt__(self, other: int | float | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.gt, other, is_comparison=True) return self._apply_op_to_all_elements(operator.gt, other, is_comparison=True) - def __ge__(self, other: Union[int, float, Array]) -> Array: + def __ge__(self, other: int | float | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.ge, other, is_comparison=True) return self._apply_op_to_all_elements(operator.ge, other, is_comparison=True) - def __le__(self, other: Union[int, float, Array]) -> Array: + def __le__(self, other: int | float | Array) -> Array: if isinstance(other, Array): return self._apply_op_between_arrays(operator.le, other, is_comparison=True) return self._apply_op_to_all_elements(operator.le, other, is_comparison=True) @@ -775,4 +838,4 @@ def __neg__(self): return self._apply_op_to_all_elements(operator.neg, None) def __abs__(self): - return self._apply_op_to_all_elements(operator.abs, None) \ No newline at end of file + return self._apply_op_to_all_elements(operator.abs, None) diff --git a/bitstring/bitarray_.py b/bitstring/bitarray_.py index a372b447..e547f5eb 100644 --- a/bitstring/bitarray_.py +++ b/bitstring/bitarray_.py @@ -1,16 +1,20 @@ from __future__ import annotations import copy +import mmap import numbers +import pathlib import re from collections import abc -from typing import Union, List, Iterable, Any, Optional +from typing import Any, BinaryIO +from collections.abc import Iterable +from tibs import Mutibs, Tibs from bitstring import utils from bitstring.exceptions import CreationError, Error from bitstring.bits import Bits, BitsType, TBits import bitstring.dtypes -common_helpers = bitstring.bitstore_common_helpers +import bitstring.bitstore_helpers as helpers MutableBitStore = bitstring.bitstore.MutableBitStore @@ -46,15 +50,16 @@ class BitArray(Bits): endswith() -- Return whether the bitstring ends with a sub-string. find() -- Find a sub-bitstring in the current bitstring. findall() -- Find all occurrences of a sub-bitstring in the current bitstring. - fromstring() -- Create a bitstring from a formatted string. + from_string() -- Create a bitstring from a formatted string. + from_tibs() -- Create a bitstring from a tibs.Tibs or tibs.Mutibs instance. join() -- Join bitstrings together using current bitstring. pp() -- Pretty print the bitstring. rfind() -- Seek backwards to find a sub-bitstring. split() -- Create generator of chunks split by a delimiter. startswith() -- Return whether the bitstring starts with a sub-bitstring. - tobitarray() -- Return bitstring as a bitarray from the bitarray package. - tobytes() -- Return bitstring as bytes, padding if needed. - tofile() -- Write bitstring to file, padding if needed. + to_bytes() -- Return bitstring as bytes, padding if needed. + to_file() -- Write bitstring to file, padding if needed. + to_tibs() -- Return the data as a tibs.Tibs instance. unpack() -- Interpret bits using format string. Special methods: @@ -75,36 +80,35 @@ class BitArray(Bits): # As BitArray objects are mutable, we shouldn't allow them to be hashed. __hash__: None = None - def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None, - offset: Optional[int] = None, **kwargs) -> None: + def __init__(self, auto: BitsType | None = None, /, length: int | None = None, + offset: int | None = None, **kwargs) -> None: """Either specify an 'auto' initialiser: - A string of comma separated tokens, an integer, a file object, - a bytearray, a boolean iterable or another bitstring. + A string of comma separated tokens, a bytes-like object or another bitstring. Or initialise via **kwargs with one (and only one) of: bin -- binary string representation, e.g. '0b001010'. hex -- hexadecimal string representation, e.g. '0x2ef' oct -- octal string representation, e.g. '0o777'. bytes -- raw data as a bytes object, for example read from a binary file. - int -- a signed integer. - uint -- an unsigned integer. - float / floatbe -- a big-endian floating point number. + i -- a signed integer. int is a compatibility alias. + u -- an unsigned integer. uint is a compatibility alias. + f / float / fbe -- a big-endian floating point number. bool -- a boolean (True or False). se -- a signed exponential-Golomb code. ue -- an unsigned exponential-Golomb code. sie -- a signed interleaved exponential-Golomb code. uie -- an unsigned interleaved exponential-Golomb code. - floatle -- a little-endian floating point number. - floatne -- a native-endian floating point number. + fle -- a little-endian floating point number. + fne -- a native-endian floating point number. bfloat / bfloatbe - a big-endian bfloat format 16-bit floating point number. bfloatle -- a little-endian bfloat format 16-bit floating point number. bfloatne -- a native-endian bfloat format 16-bit floating point number. - intbe -- a signed big-endian whole byte integer. - intle -- a signed little-endian whole byte integer. - intne -- a signed native-endian whole byte integer. - uintbe -- an unsigned big-endian whole byte integer. - uintle -- an unsigned little-endian whole byte integer. - uintne -- an unsigned native-endian whole byte integer. + ibe -- a signed big-endian whole byte integer. + ile -- a signed little-endian whole byte integer. + ine -- a signed native-endian whole byte integer. + ube -- an unsigned big-endian whole byte integer. + ule -- an unsigned little-endian whole byte integer. + une -- an unsigned native-endian whole byte integer. filename -- the path of a file which will be opened in binary read-only mode. Other keyword arguments: @@ -117,27 +121,129 @@ def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optio """ pass - def __new__(cls: Type[TBits], auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None, - offset: Optional[int] = None, **kwargs) -> TBits: + def __new__(cls: type[TBits], auto: BitsType | None = None, /, length: int | None = None, + offset: int | None = None, **kwargs) -> TBits: x = super(Bits, cls).__new__(cls) if auto is None and not kwargs: # No initialiser so fill with zero bits up to length - if length is not None: - x._bitstore = MutableBitStore.from_zeros(length) - else: - x._bitstore = MutableBitStore() + x._bitstore = MutableBitStore.from_zeros(length if length is not None else 0) return x x._initialise(auto, length, offset, immutable=False, **kwargs) return x @classmethod - def fromstring(cls: TBits, s: str, /) -> TBits: + def from_string(cls: type[TBits], s: str, /) -> TBits: """Create a new bitstring from a formatted string.""" x = super().__new__(cls) - b = common_helpers.str_to_bitstore(s) + b = helpers.str_to_bitstore(s) x._bitstore = b._mutable_copy() return x + @classmethod + def from_dtype(cls: type[TBits], dtype: str | bitstring.Dtype, value: Any, /) -> TBits: + """Create a new bitstring by packing value according to dtype.""" + x = super().__new__(cls) + x._bitstore = bitstring.dtypes.Dtype(dtype).pack(value)._bitstore._mutable_copy() + return x + + @classmethod + def from_bytes(cls: type[TBits], data: bytes | bytearray | memoryview, /, *, + length: int | None = None, offset: int = 0) -> TBits: + """Create a new bitstring from a bytes-like object.""" + x = super().__new__(cls) + x._bitstore = MutableBitStore.from_bytes(data, offset=offset, length=length) + return x + + @classmethod + def from_bools(cls: type[TBits], iterable: Iterable[Any], /) -> TBits: + """Create a new bitstring from an iterable of bool-like values.""" + x = super().__new__(cls) + x._bitstore = MutableBitStore.from_bools(iterable) + return x + + @classmethod + def from_zeros(cls: type[TBits], length: int, /) -> TBits: + """Create a new bitstring containing length zero bits.""" + length = int(length) + if length < 0: + raise bitstring.CreationError(f"Can't create bitstring of negative length {length}.") + x = super().__new__(cls) + x._bitstore = MutableBitStore.from_zeros(length) + return x + + @classmethod + def from_ones(cls: type[TBits], length: int, /) -> TBits: + """Create a new bitstring containing length one bits.""" + length = int(length) + if length < 0: + raise bitstring.CreationError(f"Can't create bitstring of negative length {length}.") + x = super().__new__(cls) + x._bitstore = MutableBitStore.from_ones(length) + return x + + @classmethod + def from_joined(cls: type[TBits], sequence: Iterable[BitsType], /) -> TBits: + """Create a new bitstring by concatenating a sequence of bitstrings.""" + sequence_iter = iter(sequence) + x = super().__new__(cls) + try: + first = Bits._create_from_bitstype(next(sequence_iter))._bitstore + except StopIteration: + x._bitstore = MutableBitStore.from_zeros(0) + return x + + def stores() -> Iterable[MutableBitStore | bitstring.bitstore.ConstBitStore]: + yield first + for item in sequence_iter: + yield Bits._create_from_bitstype(item)._bitstore + + x._bitstore = MutableBitStore.join(stores()) + return x + + @classmethod + def from_file(cls: type[TBits], source: str | pathlib.Path | BinaryIO, /, *, + length: int | None = None, offset: int = 0) -> TBits: + """Create a new bitstring from a file path or binary file object.""" + x = super().__new__(cls) + filename = source if isinstance(source, (str, pathlib.Path)) else source.name + with open(pathlib.Path(filename), 'rb') as source_file: + m = mmap.mmap(source_file.fileno(), 0, access=mmap.ACCESS_READ) + file_bits = len(m) * 8 + if offset == 0: + x._bitstore = MutableBitStore.frombuffer(m, length=length) + else: + if length is None: + if offset > file_bits: + raise bitstring.CreationError(f"The offset of {offset} bits is greater than the file length ({file_bits} bits).") + x._bitstore = MutableBitStore.frombuffer(m, offset=offset) + else: + if offset + length > file_bits: + raise bitstring.CreationError( + f"Can't use a length of {length} bits and an offset of {offset} bits as file length is only {file_bits} bits.") + x._bitstore = MutableBitStore.frombuffer(m, offset=offset, length=length) + return x + + @classmethod + def from_tibs(cls: type[TBits], tibs: Tibs | Mutibs, /) -> TBits: + """Create a new bitstring from a tibs.Tibs or tibs.Mutibs instance.""" + if isinstance(tibs, Mutibs): + tibs = tibs.to_tibs() + elif not isinstance(tibs, Tibs): + raise TypeError(f"Expected tibs.Tibs or tibs.Mutibs, got {type(tibs).__name__}.") + x = super().__new__(cls) + x._bitstore = MutableBitStore(tibs.to_mutibs()) + return x + + def to_bits(self) -> Bits: + """Return an immutable copy of the bitstring.""" + x = Bits() + x._bitstore = bitstring.bitstore.ConstBitStore(self._bitstore.tibs.to_tibs()) + return x + + def to_tibs(self) -> Tibs: + """Return the data as a tibs.Tibs instance.""" + return self._bitstore.tibs.to_tibs() + def copy(self: TBits) -> TBits: """Return a copy of the bitstring.""" return self.__copy__() @@ -159,7 +265,7 @@ def __setattr__(self, attribute, value) -> None: if len(x) != dtype.bitlength: raise CreationError(f"Can't initialise with value of length {len(x)} bits, " f"as attribute has length of {dtype.bitlength} bits.") - self._bitstore = x._bitstore + self._bitstore = x._bitstore._mutable_copy() return def __iadd__(self, bs: BitsType) -> BitArray: @@ -177,7 +283,7 @@ def __copy__(self) -> BitArray: s_copy._bitstore = self._bitstore._mutable_copy() return s_copy - def _setitem_int(self, key: int, value: Union[BitsType, int]) -> None: + def _setitem_int(self, key: int, value: BitsType | int) -> None: if isinstance(value, numbers.Integral): if value == 0: self._bitstore[key] = 0 @@ -209,9 +315,9 @@ def _setitem_slice(self, key: slice, value: BitsType) -> None: length = len(s) # Now create an int of the correct length if value >= 0: - value = self.__class__(uint=value, length=length) + value = self.__class__(u=value, length=length) else: - value = self.__class__(int=value, length=length) + value = self.__class__(i=value, length=length) else: try: value = self._create_from_bitstype(value) @@ -219,13 +325,13 @@ def _setitem_slice(self, key: slice, value: BitsType) -> None: raise TypeError(f"Bitstring, integer or string expected. Got {type(value)}.") self._bitstore.__setitem__(key, value._bitstore) - def __setitem__(self, key: Union[slice, int], value: BitsType) -> None: + def __setitem__(self, key: slice | int, value: BitsType) -> None: if isinstance(key, numbers.Integral): self._setitem_int(int(key), value) else: self._setitem_slice(key, value) - def __delitem__(self, key: Union[slice, int]) -> None: + def __delitem__(self, key: slice | int) -> None: """Delete item or range. >>> a = BitArray('0x001122') @@ -293,39 +399,15 @@ def __ixor__(self: TBits, bs: BitsType) -> TBits: self._bitstore ^= bs._bitstore return self - def _replace(self, old: Bits, new: Bits, start: int, end: int, count: int, bytealigned: Optional[bool]) -> int: + def _replace(self, old: Bits, new: Bits, start: int | None, end: int | None, + count: int | None, bytealigned: bool | None) -> int: if bytealigned is None: bytealigned = bitstring.options.bytealigned - # First find all the places where we want to do the replacements - starting_points: List[int] = [] - for x in self.findall(old, start, end, bytealigned=bytealigned): - if not starting_points: - starting_points.append(x) - elif x >= starting_points[-1] + len(old): - # Can only replace here if it hasn't already been replaced! - starting_points.append(x) - if count != 0 and len(starting_points) == count: - break - if not starting_points: - return 0 - replacement_list = [self._bitstore.getslice(0, starting_points[0])] - for i in range(len(starting_points) - 1): - replacement_list.append(new._bitstore) - replacement_list.append( - self._bitstore.getslice(starting_points[i] + len(old), starting_points[i + 1])) - # Final replacement - replacement_list.append(new._bitstore) - replacement_list.append(self._bitstore.getslice(starting_points[-1] + len(old), None)) - if bitstring.options.lsb0: - # Addition of bitarray is always on the right, so assemble from other end - replacement_list.reverse() - self._bitstore.clear() - for r in replacement_list: - self._bitstore += r - return len(starting_points) - - def replace(self, old: BitsType, new: BitsType, start: Optional[int] = None, end: Optional[int] = None, - count: Optional[int] = None, bytealigned: Optional[bool] = None) -> int: + return self._bitstore.replace(old._bitstore, new._bitstore, start=start, end=end, + count=count, bytealigned=bytealigned) + + def replace(self, old: BitsType, new: BitsType, *, start: int | None = None, end: int | None = None, + count: int | None = None, bytealigned: bool | None = None) -> int: """Replace all occurrences of old with new in place. Returns number of replacements made. @@ -351,12 +433,11 @@ def replace(self, old: BitsType, new: BitsType, start: Optional[int] = None, end new = self._create_from_bitstype(new) if len(old) == 0: raise ValueError("Empty bitstring cannot be replaced.") - start, end = self._validate_slice(start, end) if new is self: # Prevent self assignment woes new = copy.copy(self) - return self._replace(old, new, start, end, 0 if count is None else count, bytealigned) + return self._replace(old, new, start, end, count, bytealigned) def insert(self, bs: BitsType, pos: int) -> None: """Insert bs at bit position pos. @@ -412,14 +493,22 @@ def prepend(self, bs: BitsType) -> None: """ self._prepend(bs) - def _append_msb0(self, bs: BitsType) -> None: + def _append(self, bs: BitsType) -> None: self._addright(self._create_from_bitstype(bs)) - def _append_lsb0(self, bs: BitsType) -> None: + def _prepend(self, bs: BitsType) -> None: bs = self._create_from_bitstype(bs) self._addleft(bs) - def reverse(self, start: Optional[int] = None, end: Optional[int] = None) -> None: + @staticmethod + def _normalise_positions(pos: int | Iterable[int]) -> int | Iterable[int]: + if not isinstance(pos, abc.Iterable): + return pos + if isinstance(pos, (list, tuple, range)): + return pos + return tuple(pos) + + def reverse(self, *, start: int | None = None, end: int | None = None) -> None: """Reverse bits in-place. start -- Position of first bit to reverse. Defaults to 0. @@ -439,7 +528,7 @@ def reverse(self, start: Optional[int] = None, end: Optional[int] = None) -> Non s._bitstore.reverse() self[start:end] = s - def set(self, value: Any, pos: Optional[Union[int, Iterable[int]]] = None) -> None: + def set(self, value: Any, pos: int | Iterable[int] | None = None) -> None: """Set one or many bits to 1 or 0. value -- If bool(value) is True bits are set to 1, otherwise they are set to 0. @@ -454,16 +543,9 @@ def set(self, value: Any, pos: Optional[Union[int, Iterable[int]]] = None) -> No # Set all bits to either 1 or 0 self._bitstore.__setitem__(slice(None), bool(value)) return - if not isinstance(pos, abc.Iterable): - pos = (pos,) - v = 1 if value else 0 - if isinstance(pos, range): - self._bitstore.__setitem__(slice(pos.start, pos.stop, pos.step), v) - return - for p in pos: - self._bitstore[p] = v + self._bitstore.set(value, BitArray._normalise_positions(pos)) - def invert(self, pos: Optional[Union[Iterable[int], int]] = None) -> None: + def invert(self, pos: Iterable[int] | int | None = None) -> None: """Invert one or many bits from 0 to 1 or vice versa. pos -- Either a single bit position or an iterable of bit positions. @@ -475,18 +557,9 @@ def invert(self, pos: Optional[Union[Iterable[int], int]] = None) -> None: if pos is None: self._invert_all() return - if not isinstance(pos, abc.Iterable): - pos = (pos,) - length = len(self) - - for p in pos: - if p < 0: - p += length - if not 0 <= p < length: - raise IndexError(f"Bit position {p} out of range.") - self._invert(p) + self._bitstore.invert(BitArray._normalise_positions(pos)) - def ror(self, bits: int, start: Optional[int] = None, end: Optional[int] = None) -> None: + def ror(self, bits: int, *, start: int | None = None, end: int | None = None) -> None: """Rotate bits to the right in-place. bits -- The number of bits to rotate by. @@ -502,16 +575,14 @@ def ror(self, bits: int, start: Optional[int] = None, end: Optional[int] = None) raise ValueError("Cannot rotate by negative amount.") self._ror(bits, start, end) - def _ror_msb0(self, bits: int, start: Optional[int] = None, end: Optional[int] = None) -> None: - start, end = self._validate_slice(start, end) # the _slice deals with msb0/lsb0 + def _ror(self, bits: int, start: int | None = None, end: int | None = None) -> None: + start, end = self._validate_slice(start, end) bits %= (end - start) if not bits: return - rhs = self._slice(end - bits, end) - self._delete(bits, end - bits) - self._insert(rhs, start) + self._bitstore.rotate_right(bits, start=start, end=end) - def rol(self, bits: int, start: Optional[int] = None, end: Optional[int] = None) -> None: + def rol(self, bits: int, *, start: int | None = None, end: int | None = None) -> None: """Rotate bits to the left in-place. bits -- The number of bits to rotate by. @@ -527,17 +598,15 @@ def rol(self, bits: int, start: Optional[int] = None, end: Optional[int] = None) raise ValueError("Cannot rotate by negative amount.") self._rol(bits, start, end) - def _rol_msb0(self, bits: int, start: Optional[int] = None, end: Optional[int] = None): + def _rol(self, bits: int, start: int | None = None, end: int | None = None): start, end = self._validate_slice(start, end) bits %= (end - start) if bits == 0: return - lhs = self._slice(start, start + bits) - self._delete(bits, start) - self._insert(lhs, end - bits) + self._bitstore.rotate_left(bits, start=start, end=end) - def byteswap(self, fmt: Optional[Union[int, Iterable[int], str]] = None, start: Optional[int] = None, - end: Optional[int] = None, repeat: bool = True) -> int: + def byteswap(self, fmt: int | Iterable[int] | str | None = None, start: int | None = None, + end: int | None = None, repeat: bool = True) -> int: """Change the endianness in-place. Return number of repeats of fmt done. fmt -- A compact structure string, an integer number of bytes or diff --git a/bitstring/bits.py b/bitstring/bits.py index 5ef096a1..60152eeb 100644 --- a/bitstring/bits.py +++ b/bitstring/bits.py @@ -9,8 +9,9 @@ import io from collections import abc import functools -from typing import Tuple, Union, List, Iterable, Any, Optional, BinaryIO, TextIO, overload, Iterator, Type, TypeVar -import bitarray +from typing import Union, Any, BinaryIO, TextIO, overload, TypeVar +from collections.abc import Iterable, Iterator +from tibs import Mutibs, Tibs import bitstring from bitstring import utils from bitstring.dtypes import Dtype, dtype_register @@ -18,14 +19,15 @@ from bitstring.mxfp import e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt, e4m3mxfp_saturate_fmt, e5m2mxfp_saturate_fmt from bitstring.bitstring_options import Colour +import bitstring.bitstore_helpers as helpers + ConstBitStore = bitstring.bitstore.ConstBitStore MutableBitStore = bitstring.bitstore.MutableBitStore -helpers = bitstring.bitstore_helpers -common_helpers = bitstring.bitstore_common_helpers + # Things that can be converted to Bits when a Bits type is needed -BitsType = Union['Bits', str, Iterable[Any], bool, BinaryIO, bytearray, bytes, memoryview, bitarray.bitarray] +BitsType = Union['Bits', str, Tibs, Mutibs, Iterable[Any], BinaryIO, bytearray, bytes, memoryview] TBits = TypeVar("TBits", bound='Bits') @@ -48,15 +50,16 @@ class Bits: endswith() -- Return whether the bitstring ends with a sub-string. find() -- Find a sub-bitstring in the current bitstring. findall() -- Find all occurrences of a sub-bitstring in the current bitstring. - fromstring() -- Create a bitstring from a formatted string. + from_string() -- Create a bitstring from a formatted string. + from_tibs() -- Create a bitstring from a tibs.Tibs or tibs.Mutibs instance. join() -- Join bitstrings together using current bitstring. pp() -- Pretty print the bitstring. rfind() -- Seek backwards to find a sub-bitstring. split() -- Create generator of chunks split by a delimiter. startswith() -- Return whether the bitstring starts with a sub-bitstring. - tobitarray() -- Return bitstring as a bitarray from the bitarray package. - tobytes() -- Return bitstring as bytes, padding if needed. - tofile() -- Write bitstring to file, padding if needed. + to_bytes() -- Return bitstring as bytes, padding if needed. + to_file() -- Write bitstring to file, padding if needed. + to_tibs() -- Return the data as a tibs.Tibs instance. unpack() -- Interpret bits using format string. Special methods: @@ -72,36 +75,35 @@ class Bits: """ __slots__ = ('_bitstore', '_filename') - def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None, - offset: Optional[int] = None, **kwargs) -> None: + def __init__(self, auto: BitsType | None = None, /, length: int | None = None, + offset: int | None = None, **kwargs) -> None: """Either specify an 'auto' initialiser: - A string of comma separated tokens, an integer, a file object, - a bytearray, a boolean iterable, an array or another bitstring. + A string of comma separated tokens, a bytes-like object or another bitstring. Or initialise via **kwargs with one (and only one) of: bin -- binary string representation, e.g. '0b001010'. hex -- hexadecimal string representation, e.g. '0x2ef' oct -- octal string representation, e.g. '0o777'. bytes -- raw data as a bytes object, for example read from a binary file. - int -- a signed integer. - uint -- an unsigned integer. - float / floatbe -- a big-endian floating point number. + i -- a signed integer. int is a compatibility alias. + u -- an unsigned integer. uint is a compatibility alias. + f / float / fbe -- a big-endian floating point number. bool -- a boolean (True or False). se -- a signed exponential-Golomb code. ue -- an unsigned exponential-Golomb code. sie -- a signed interleaved exponential-Golomb code. uie -- an unsigned interleaved exponential-Golomb code. - floatle -- a little-endian floating point number. - floatne -- a native-endian floating point number. + fle -- a little-endian floating point number. + fne -- a native-endian floating point number. bfloat / bfloatbe - a big-endian bfloat format 16-bit floating point number. bfloatle -- a little-endian bfloat format 16-bit floating point number. bfloatne -- a native-endian bfloat format 16-bit floating point number. - intbe -- a signed big-endian whole byte integer. - intle -- a signed little-endian whole byte integer. - intne -- a signed native-endian whole byte integer. - uintbe -- an unsigned big-endian whole byte integer. - uintle -- an unsigned little-endian whole byte integer. - uintne -- an unsigned native-endian whole byte integer. + ibe -- a signed big-endian whole byte integer. + ile -- a signed little-endian whole byte integer. + ine -- a signed native-endian whole byte integer. + ube -- an unsigned big-endian whole byte integer. + ule -- an unsigned little-endian whole byte integer. + une -- an unsigned native-endian whole byte integer. filename -- the path of a file which will be opened in binary read-only mode. Other keyword arguments: @@ -114,38 +116,52 @@ def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optio """ pass - def __new__(cls: Type[TBits], auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None, - offset: Optional[int] = None, pos: Optional[int] = None, **kwargs) -> TBits: + def __new__(cls: type[TBits], auto: BitsType | None = None, /, length: int | None = None, + offset: int | None = None, **kwargs) -> TBits: x = super().__new__(cls) if auto is None and not kwargs: # No initialiser so fill with zero bits up to length - if length is not None: - x._bitstore = ConstBitStore.from_zeros(length) - else: - x._bitstore = ConstBitStore() + x._bitstore = ConstBitStore.from_zeros(length if length is not None else 0) return x x._initialise(auto, length, offset, immutable=True, **kwargs) return x @classmethod - def _create_from_bitstype(cls: Type[TBits], auto: BitsType, /) -> TBits: + def _create_from_bitstype(cls: type[TBits], auto: BitsType, /) -> TBits: if isinstance(auto, cls): return auto b = super().__new__(cls) b._setauto_no_length_or_offset(auto) return b - def _initialise(self, auto: Any, /, length: Optional[int], offset: Optional[int], immutable: bool, **kwargs) -> None: + def _initialise(self, auto: Any, /, length: int | None, offset: int | None, immutable: bool, **kwargs) -> None: if auto is not None: if isinstance(auto, numbers.Integral): - # Initialise with s zero bits. - if auto < 0: - raise bitstring.CreationError(f"Can't create bitstring of negative length {auto}.") - if immutable: - self._bitstore = ConstBitStore.from_zeros(int(auto)) - else: - self._bitstore = MutableBitStore.from_zeros(int(auto)) - return + raise TypeError( + f"It's no longer possible to initialise a bitstring from an integer. " + f"Use '{self.__class__.__name__}.from_zeros({int(auto)})' instead of " + f"'{self.__class__.__name__}({int(auto)})'." + ) + if isinstance(auto, io.BytesIO): + raise TypeError( + f"It's no longer possible to initialise a bitstring directly from a BytesIO object. " + f"Use '{self.__class__.__name__}.from_bytes(bytes_io.getvalue())' instead." + ) + if isinstance(auto, io.BufferedReader): + raise TypeError( + f"It's no longer possible to initialise a bitstring directly from a file object. " + f"Use '{self.__class__.__name__}.from_file(file)' instead." + ) + if isinstance(auto, array.array): + raise TypeError( + f"It's no longer possible to initialise a bitstring directly from an array object. " + f"Use '{self.__class__.__name__}.from_bytes(array_obj.tobytes())' instead." + ) + if isinstance(auto, abc.Iterable) and not isinstance(auto, (str, Bits, Tibs, Mutibs, bytes, bytearray, memoryview)): + raise TypeError( + f"It's no longer possible to initialise a bitstring directly from an arbitrary iterable. " + f"Use '{self.__class__.__name__}.from_bools(iterable)' instead." + ) self._setauto(auto, length, offset) else: k, v = kwargs.popitem() @@ -154,8 +170,6 @@ def _initialise(self, auto: Any, /, length: Optional[int], offset: Optional[int] self._setbytes_with_truncation(v, length, offset) elif k == 'filename': self._setfile(v, length, offset) - elif k == 'bitarray': - self._setbitarray(v, length, offset) elif k == 'auto': raise bitstring.CreationError( f"The 'auto' parameter should not be given explicitly - just use the first positional argument. " @@ -167,7 +181,10 @@ def _initialise(self, auto: Any, /, length: Optional[int], offset: Optional[int] Dtype(k, length).set_fn(self, v) except ValueError as e: raise bitstring.CreationError(e) - if not immutable: + if immutable: + if isinstance(self._bitstore, MutableBitStore): + self._bitstore = ConstBitStore(self._bitstore.tibs.to_tibs()) + else: # TODO: This copy is not a good idea. self._bitstore = self._bitstore._mutable_copy() @@ -210,11 +227,8 @@ def __add__(self: TBits, bs: BitsType) -> TBits: """ bs = self.__class__._create_from_bitstype(bs) - s = self._copy() if len(bs) <= len(self) else bs._copy() - if len(bs) <= len(self): - s._addright(bs) - else: - s._addleft(self) + s = super().__new__(self.__class__) + s._bitstore = self._bitstore + bs._bitstore return s def __radd__(self: TBits, bs: BitsType) -> TBits: @@ -234,7 +248,7 @@ def __getitem__(self: TBits, key: slice, /) -> TBits: def __getitem__(self, key: int, /) -> bool: ... - def __getitem__(self: TBits, key: Union[slice, int], /) -> Union[TBits, bool]: + def __getitem__(self: TBits, key: slice | int, /) -> TBits | bool: """Return a new bitstring representing a slice of the current bitstring. >>> print(Bits('0b00110')[1:4]) @@ -252,7 +266,7 @@ def __len__(self) -> int: return self._getlength() def __bytes__(self) -> bytes: - return self.tobytes() + return self.to_bytes() def __str__(self) -> str: """Return approximate string representation of bitstring for printing. @@ -342,9 +356,8 @@ def __lshift__(self: TBits, n: int, /) -> TBits: raise ValueError("Cannot shift by a negative amount.") if len(self) == 0: raise ValueError("Cannot shift an empty bitstring.") - n = min(n, len(self)) - s = self._absolute_slice(n, len(self)) - s._addright(Bits(n)) + s = object.__new__(self.__class__) + s._bitstore = self._bitstore << n return s def __rshift__(self: TBits, n: int, /) -> TBits: @@ -357,11 +370,8 @@ def __rshift__(self: TBits, n: int, /) -> TBits: raise ValueError("Cannot shift by a negative amount.") if len(self) == 0: raise ValueError("Cannot shift an empty bitstring.") - if not n: - return self._copy() - s = self.__class__(length=min(n, len(self))) - n = min(n, len(self)) - s._addright(self._absolute_slice(0, len(self) - n)) + s = object.__new__(self.__class__) + s._bitstore = self._bitstore >> n return s def __mul__(self: TBits, n: int, /) -> TBits: @@ -381,7 +391,7 @@ def __mul__(self: TBits, n: int, /) -> TBits: def _imul(self: TBits, n: int, /) -> TBits: """Concatenate n copies of self in place. Return self.""" - self._bitstore.__imul__(n) + self._bitstore.tibs *= n return self def __rmul__(self: TBits, n: int, /) -> TBits: @@ -473,7 +483,7 @@ def __contains__(self, bs: BitsType, /) -> bool: """ found = Bits.find(self, bs, bytealigned=False) - return bool(found) + return found is not None def __hash__(self) -> int: """Return an integer hash of the object.""" @@ -483,11 +493,11 @@ def __hash__(self) -> int: # bit position inside the bitstring as that does not feature in the __eq__ operation. if len(self) <= 2000: # Use the whole bitstring. - return hash((self.tobytes(), len(self))) + return hash((self.to_bytes(), len(self))) else: # We can't in general hash the whole bitstring (it could take hours!) # So instead take some bits from the start and end. - return hash(((self[:800] + self[-800:]).tobytes(), len(self))) + return hash(((self[:800] + self[-800:]).to_bytes(), len(self))) def __bool__(self) -> bool: """Return False if bitstring is empty, otherwise return True.""" @@ -498,33 +508,36 @@ def _clear(self) -> None: self._bitstore.clear() def _setauto_no_length_or_offset(self, s: BitsType, /) -> None: - """Set bitstring from a bitstring, file, bool, array, iterable or string.""" + """Set bitstring from a bitstring, file, array, iterable or string.""" if isinstance(s, str): - self._bitstore = common_helpers.str_to_bitstore(s) + self._bitstore = helpers.str_to_bitstore(s) elif isinstance(s, Bits): self._bitstore = s._bitstore.copy() + elif isinstance(s, Tibs): + self._bitstore = ConstBitStore(s) + elif isinstance(s, Mutibs): + self._bitstore = ConstBitStore(s.to_tibs()) elif isinstance(s, (bytes, bytearray, memoryview)): - self._bitstore = ConstBitStore.from_bytes(bytearray(s)) + self._bitstore = ConstBitStore.from_bytes(s) elif isinstance(s, io.BytesIO): self._bitstore = ConstBitStore.from_bytes(s.getvalue()) elif isinstance(s, io.BufferedReader): self._setfile(s.name) - elif isinstance(s, bitarray.bitarray): - self._bitstore = ConstBitStore(s) elif isinstance(s, array.array): self._bitstore = ConstBitStore.from_bytes(s.tobytes()) elif isinstance(s, abc.Iterable): - # Evaluate each item as True or False and set bits to 1 or 0. - self._setbin(''.join(str(int(bool(x))) for x in s)) + self._bitstore = ConstBitStore.from_bools(s) elif isinstance(s, numbers.Integral): - raise TypeError(f"It's no longer possible to auto initialise a bitstring from an integer." - f" Use '{self.__class__.__name__}({s})' instead of just '{s}' as this makes it " - f"clearer that a bitstring of {int(s)} zero bits will be created.") + raise TypeError( + f"It's no longer possible to initialise a bitstring from an integer. " + f"Use '{self.__class__.__name__}.from_zeros({int(s)})' instead of " + f"'{self.__class__.__name__}({int(s)})'." + ) else: raise TypeError(f"Cannot initialise bitstring from type '{type(s)}'.") - def _setauto(self, s: BitsType, length: Optional[int], offset: Optional[int], /) -> None: - """Set bitstring from a bitstring, file, bool, array, iterable or string.""" + def _setauto(self, s: BitsType, length: int | None, offset: int | None, /) -> None: + """Set bitstring from a bitstring, file, array, iterable or string.""" # As s can be so many different things it's important to do the checks # in the correct order, as some types are also other allowed types. if offset is None and length is None: @@ -534,14 +547,12 @@ def _setauto(self, s: BitsType, length: Optional[int], offset: Optional[int], /) offset = 0 if isinstance(s, io.BytesIO): + total_bits = s.seek(0, 2) * 8 if length is None: - length = s.seek(0, 2) * 8 - offset - byteoffset, offset = divmod(offset, 8) - bytelength = (length + byteoffset * 8 + offset + 7) // 8 - byteoffset - if length + byteoffset * 8 + offset > s.seek(0, 2) * 8: + length = total_bits - offset + if length + offset > total_bits: raise bitstring.CreationError("BytesIO object is not long enough for specified length and offset.") - self._bitstore = ConstBitStore.from_bytes(s.getvalue()[byteoffset: byteoffset + bytelength]).getslice( - offset, offset + length) + self._bitstore = ConstBitStore.from_bytes(s.getvalue(), offset=offset, length=length) return if isinstance(s, io.BufferedReader): @@ -549,84 +560,76 @@ def _setauto(self, s: BitsType, length: Optional[int], offset: Optional[int], /) return if isinstance(s, (str, Bits, bytes, bytearray, memoryview, io.BytesIO, io.BufferedReader, - bitarray.bitarray, array.array, abc.Iterable)): + array.array, abc.Iterable)): raise bitstring.CreationError(f"Cannot initialise bitstring from type '{type(s)}' when using explicit lengths or offsets.") raise TypeError(f"Cannot initialise bitstring from type '{type(s)}'.") - def _setfile(self, filename: str, length: Optional[int] = None, offset: Optional[int] = None) -> None: + def _setfile(self, filename: str, length: int | None = None, offset: int | None = None) -> None: """Use file as source of bits.""" with open(pathlib.Path(filename), 'rb') as source: if offset is None: offset = 0 m = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ) + file_bits = len(m) * 8 if offset == 0: self._filename = source.name self._bitstore = ConstBitStore.frombuffer(m, length=length) else: - # If offset is given then always read into memory. - temp = ConstBitStore.frombuffer(m) if length is None: - if offset > len(temp): - raise bitstring.CreationError(f"The offset of {offset} bits is greater than the file length ({len(temp)} bits).") - self._bitstore = temp.getslice(offset, None) + if offset > file_bits: + raise bitstring.CreationError(f"The offset of {offset} bits is greater than the file length ({file_bits} bits).") + self._bitstore = ConstBitStore.frombuffer(m, offset=offset) else: - self._bitstore = temp.getslice(offset, offset + length) - if len(self) != length: - raise bitstring.CreationError(f"Can't use a length of {length} bits and an offset of {offset} bits as file length is only {len(temp)} bits.") - - def _setbitarray(self, ba: bitarray.bitarray, length: Optional[int], offset: Optional[int]) -> None: - if offset is None: - offset = 0 - if offset > len(ba): - raise bitstring.CreationError(f"Offset of {offset} too large for bitarray of length {len(ba)}.") - if length is None: - self._bitstore = ConstBitStore(ba[offset:]) - else: - if offset + length > len(ba): - raise bitstring.CreationError( - f"Offset of {offset} and length of {length} too large for bitarray of length {len(ba)}.") - self._bitstore = ConstBitStore(ba[offset: offset + length]) + if offset + length > file_bits: + raise bitstring.CreationError( + f"Can't use a length of {length} bits and an offset of {offset} bits as file length is only {file_bits} bits.") + self._bitstore = ConstBitStore.frombuffer(m, offset=offset, length=length) def _setbits(self, bs: BitsType, length: None = None) -> None: bs = Bits._create_from_bitstype(bs) self._bitstore = bs._bitstore def _setp3binary(self, f: float) -> None: - self._bitstore = common_helpers.p3binary2bitstore(f) + self._bitstore = helpers.p3binary2bitstore(f) def _setp4binary(self, f: float) -> None: - self._bitstore = common_helpers.p4binary2bitstore(f) + self._bitstore = helpers.p4binary2bitstore(f) + + def _sete4m3mxfp_saturate(self, f: float) -> None: + self._bitstore = helpers.e4m3mxfp_saturate2bitstore(f) - def _sete4m3mxfp(self, f: float) -> None: - self._bitstore = common_helpers.e4m3mxfp2bitstore(f) + def _sete4m3mxfp_overflow(self, f: float) -> None: + self._bitstore = helpers.e4m3mxfp_overflow2bitstore(f) - def _sete5m2mxfp(self, f: float) -> None: - self._bitstore = common_helpers.e5m2mxfp2bitstore(f) + def _sete5m2mxfp_saturate(self, f: float) -> None: + self._bitstore = helpers.e5m2mxfp_saturate2bitstore(f) + + def _sete5m2mxfp_overflow(self, f: float) -> None: + self._bitstore = helpers.e5m2mxfp_overflow2bitstore(f) def _sete3m2mxfp(self, f: float) -> None: - self._bitstore = common_helpers.e3m2mxfp2bitstore(f) + self._bitstore = helpers.e3m2mxfp2bitstore(f) def _sete2m3mxfp(self, f: float) -> None: - self._bitstore = common_helpers.e2m3mxfp2bitstore(f) + self._bitstore = helpers.e2m3mxfp2bitstore(f) def _sete2m1mxfp(self, f: float) -> None: - self._bitstore = common_helpers.e2m1mxfp2bitstore(f) + self._bitstore = helpers.e2m1mxfp2bitstore(f) def _sete8m0mxfp(self, f: float) -> None: - self._bitstore = common_helpers.e8m0mxfp2bitstore(f) + self._bitstore = helpers.e8m0mxfp2bitstore(f) def _setmxint(self, f: float) -> None: - self._bitstore = common_helpers.mxint2bitstore(f) + self._bitstore = helpers.mxint2bitstore(f) - def _setbytes(self, data: Union[bytearray, bytes, List], length:None = None) -> None: + def _setbytes(self, data: bytearray | bytes | list, length:None = None) -> None: """Set the data from a bytes or bytearray object.""" self._bitstore = ConstBitStore.from_bytes(bytes(data)) - def _setbytes_with_truncation(self, data: Union[bytearray, bytes], length: Optional[int] = None, offset: Optional[int] = None) -> None: + def _setbytes_with_truncation(self, data: bytearray | bytes, length: int | None = None, offset: int | None = None) -> None: """Set the data from a bytes or bytearray object, with optional offset and length truncations.""" if offset is None and length is None: return self._setbytes(data) - data = bytearray(data) if offset is None: offset = 0 if length is None: @@ -635,7 +638,7 @@ def _setbytes_with_truncation(self, data: Union[bytearray, bytes], length: Optio else: if length + offset > len(data) * 8: raise bitstring.CreationError(f"Not enough data present. Need {length + offset} bits, have {len(data) * 8}.") - self._bitstore = ConstBitStore.from_bytes(data).getslice_msb0(offset, offset + length) + self._bitstore = ConstBitStore.from_bytes(data, offset=offset, length=length) def _getbytes(self) -> bytes: """Return the data as an ordinary bytes object.""" @@ -643,6 +646,9 @@ def _getbytes(self) -> bytes: raise bitstring.InterpretError("Cannot interpret as bytes unambiguously - not multiple of 8 bits.") return self._bitstore.to_bytes() + def _readbytes(self, pos: int, length: int) -> bytes: + return self._bitstore.read_bytes(pos, length) + _unprintable = list(range(0x00, 0x20)) # ASCII control characters _unprintable.extend(range(0x7f, 0xff)) # DEL char + non-ASCII @@ -653,7 +659,7 @@ def _getbytes_printable(self) -> str: string = ''.join(chr(0x100 + x) if x in Bits._unprintable else chr(x) for x in bytes_) return string - def _setuint(self, uint: int, length: Optional[int] = None) -> None: + def _setuint(self, uint: int, length: int | None = None) -> None: """Reset the bitstring to have given unsigned int interpretation.""" # If no length given, and we've previously been given a length, use it. if length is None and hasattr(self, 'len') and len(self) != 0: @@ -668,7 +674,10 @@ def _getuint(self) -> int: raise bitstring.InterpretError("Cannot interpret a zero length bitstring as an integer.") return self._bitstore.to_u() - def _setint(self, int_: int, length: Optional[int] = None) -> None: + def _readuint(self, pos: int, length: int) -> int: + return self._bitstore.read_u(pos, length) + + def _setint(self, int_: int, length: int | None = None) -> None: """Reset the bitstring to have given signed int interpretation.""" # If no length given, and we've previously been given a length, use it. if length is None and hasattr(self, 'len') and len(self) != 0: @@ -683,12 +692,15 @@ def _getint(self) -> int: raise bitstring.InterpretError("Cannot interpret bitstring without a length as an integer.") return self._bitstore.to_i() - def _setuintbe(self, uintbe: int, length: Optional[int] = None) -> None: + def _readint(self, pos: int, length: int) -> int: + return self._bitstore.read_i(pos, length) + + def _setuintbe(self, uintbe: int, length: int | None = None) -> None: """Set the bitstring to a big-endian unsigned int interpretation.""" if length is None and hasattr(self, 'len') and len(self) != 0: length = len(self) if length is None or length == 0: - raise bitstring.CreationError("A non-zero length must be specified with a uintbe initialiser.") + raise bitstring.CreationError("A non-zero length must be specified with a ube initialiser.") self._bitstore = helpers.int2bitstore(uintbe, length, False) def _getuintbe(self) -> int: @@ -697,12 +709,15 @@ def _getuintbe(self) -> int: raise bitstring.InterpretError(f"Big-endian integers must be whole-byte. Length = {len(self)} bits.") return self._getuint() - def _setintbe(self, intbe: int, length: Optional[int] = None) -> None: + def _readuintbe(self, pos: int, length: int) -> int: + return self._readuint(pos, length) + + def _setintbe(self, intbe: int, length: int | None = None) -> None: """Set bitstring to a big-endian signed int interpretation.""" if length is None and hasattr(self, 'len') and len(self) != 0: length = len(self) if length is None or length == 0: - raise bitstring.CreationError("A non-zero length must be specified with a intbe initialiser.") + raise bitstring.CreationError("A non-zero length must be specified with an ibe initialiser.") self._bitstore = helpers.int2bitstore(intbe, length, True) def _getintbe(self) -> int: @@ -711,80 +726,125 @@ def _getintbe(self) -> int: raise bitstring.InterpretError(f"Big-endian integers must be whole-byte. Length = {len(self)} bits.") return self._getint() - def _setuintle(self, uintle: int, length: Optional[int] = None) -> None: + def _readintbe(self, pos: int, length: int) -> int: + return self._readint(pos, length) + + def _setuintle(self, uintle: int, length: int | None = None) -> None: if length is None and hasattr(self, 'len') and len(self) != 0: length = len(self) if length is None or length == 0: - raise bitstring.CreationError("A non-zero length must be specified with a uintle initialiser.") + raise bitstring.CreationError("A non-zero length must be specified with a ule initialiser.") self._bitstore = helpers.intle2bitstore(uintle, length, False) def _getuintle(self) -> int: """Interpret as a little-endian unsigned int.""" if len(self) % 8: raise bitstring.InterpretError(f"Little-endian integers must be whole-byte. Length = {len(self)} bits.") - bs = ConstBitStore.from_bytes(self._bitstore.to_bytes()[::-1]) - return bs.to_u() + return self._bitstore.byte_swapped().to_u() + + def _readuintle(self, pos: int, length: int) -> int: + return self._bitstore.getslice(pos, pos + length).byte_swapped().to_u() - def _setintle(self, intle: int, length: Optional[int] = None) -> None: + def _setintle(self, intle: int, length: int | None = None) -> None: if length is None and hasattr(self, 'len') and len(self) != 0: length = len(self) if length is None or length == 0: - raise bitstring.CreationError("A non-zero length must be specified with an intle initialiser.") + raise bitstring.CreationError("A non-zero length must be specified with an ile initialiser.") self._bitstore = helpers.intle2bitstore(intle, length, True) def _getintle(self) -> int: """Interpret as a little-endian signed int.""" if len(self) % 8: raise bitstring.InterpretError(f"Little-endian integers must be whole-byte. Length = {len(self)} bits.") - bs = ConstBitStore.from_bytes(self._bitstore.to_bytes()[::-1]) - return bs.to_i() + return self._bitstore.byte_swapped().to_i() + + def _readintle(self, pos: int, length: int) -> int: + return self._bitstore.getslice(pos, pos + length).byte_swapped().to_i() def _getp4binary(self) -> float: u = self._getuint() return p4binary_fmt.lut_binary8_to_float[u] + def _readp4binary(self, pos: int, length: int) -> float: + u = self._readuint(pos, length) + return p4binary_fmt.lut_binary8_to_float[u] + def _getp3binary(self) -> float: u = self._getuint() return p3binary_fmt.lut_binary8_to_float[u] + def _readp3binary(self, pos: int, length: int) -> float: + u = self._readuint(pos, length) + return p3binary_fmt.lut_binary8_to_float[u] + def _gete4m3mxfp(self) -> float: u = self._getuint() return e4m3mxfp_saturate_fmt.lut_int_to_float[u] + def _reade4m3mxfp(self, pos: int, length: int) -> float: + u = self._readuint(pos, length) + return e4m3mxfp_saturate_fmt.lut_int_to_float[u] + def _gete5m2mxfp(self) -> float: u = self._getuint() return e5m2mxfp_saturate_fmt.lut_int_to_float[u] + def _reade5m2mxfp(self, pos: int, length: int) -> float: + u = self._readuint(pos, length) + return e5m2mxfp_saturate_fmt.lut_int_to_float[u] + def _gete3m2mxfp(self) -> float: u = self._getuint() return e3m2mxfp_fmt.lut_int_to_float[u] + def _reade3m2mxfp(self, pos: int, length: int) -> float: + u = self._readuint(pos, length) + return e3m2mxfp_fmt.lut_int_to_float[u] + def _gete2m3mxfp(self) -> float: u = self._getuint() return e2m3mxfp_fmt.lut_int_to_float[u] + def _reade2m3mxfp(self, pos: int, length: int) -> float: + u = self._readuint(pos, length) + return e2m3mxfp_fmt.lut_int_to_float[u] + def _gete2m1mxfp(self) -> float: u = self._getuint() return e2m1mxfp_fmt.lut_int_to_float[u] + def _reade2m1mxfp(self, pos: int, length: int) -> float: + u = self._readuint(pos, length) + return e2m1mxfp_fmt.lut_int_to_float[u] + def _gete8m0mxfp(self) -> float: u = self._getuint() - 127 if u == 128: return float('nan') return 2.0 ** u + def _reade8m0mxfp(self, pos: int, length: int) -> float: + u = self._readuint(pos, length) - 127 + if u == 128: + return float('nan') + return 2.0 ** u + def _getmxint(self) -> float: u = self._getint() return float(u) * 2 ** -6 - def _setfloat(self, f: float, length: Optional[int], big_endian: bool) -> None: + def _readmxint(self, pos: int, length: int) -> float: + u = self._readint(pos, length) + return float(u) * 2 ** -6 + + def _setfloat(self, f: float, length: int | None, big_endian: bool) -> None: if length is None and hasattr(self, 'len') and len(self) != 0: length = len(self) if length is None or length not in [16, 32, 64]: raise bitstring.CreationError("A length of 16, 32, or 64 must be specified with a float initialiser.") self._bitstore = helpers.float2bitstore(f, length, big_endian) - def _setfloatbe(self, f: float, length: Optional[int] = None) -> None: + def _setfloatbe(self, f: float, length: int | None = None) -> None: self._setfloat(f, length, True) def _getfloatbe(self) -> float: @@ -792,7 +852,11 @@ def _getfloatbe(self) -> float: fmt = {16: '>e', 32: '>f', 64: '>d'}[len(self)] return struct.unpack(fmt, self._bitstore.to_bytes())[0] - def _setfloatle(self, f: float, length: Optional[int] = None) -> None: + def _readfloatbe(self, pos: int, length: int) -> float: + fmt = {16: '>e', 32: '>f', 64: '>d'}[length] + return struct.unpack(fmt, self._bitstore.read_bytes(pos, length))[0] + + def _setfloatle(self, f: float, length: int | None = None) -> None: self._setfloat(f, length, False) def _getfloatle(self) -> float: @@ -800,23 +864,35 @@ def _getfloatle(self) -> float: fmt = {16: ' float: + fmt = {16: ' float: - zero_padded = self + Bits(16) + zero_padded = self + Bits.from_zeros(16) return zero_padded._getfloatbe() - def _setbfloatbe(self, f: Union[float, str], length: Optional[int] = None) -> None: + def _readbfloatbe(self, pos: int, length: int) -> float: + b = self._bitstore.read_bytes(pos, length) + b'\x00\x00' + return struct.unpack('>f', b)[0] + + def _setbfloatbe(self, f: float | str, length: int | None = None) -> None: if length is not None and length != 16: raise bitstring.CreationError(f"bfloats must be length 16, received a length of {length} bits.") - self._bitstore = common_helpers.bfloat2bitstore(f, True) + self._bitstore = helpers.bfloat2bitstore(f, True) def _getbfloatle(self) -> float: - zero_padded = Bits(16) + self + zero_padded = Bits.from_zeros(16) + self return zero_padded._getfloatle() - def _setbfloatle(self, f: Union[float, str], length: Optional[int] = None) -> None: + def _readbfloatle(self, pos: int, length: int) -> float: + b = b'\x00\x00' + self._bitstore.read_bytes(pos, length) + return struct.unpack(' None: if length is not None and length != 16: raise bitstring.CreationError(f"bfloats must be length 16, received a length of {length} bits.") - self._bitstore = common_helpers.bfloat2bitstore(f, False) + self._bitstore = helpers.bfloat2bitstore(f, False) def _setue(self, i: int) -> None: """Initialise bitstring with unsigned exponential-Golomb code for integer i. @@ -824,19 +900,15 @@ def _setue(self, i: int) -> None: Raises CreationError if i < 0. """ - if bitstring.options.lsb0: - raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.") - self._bitstore = common_helpers.ue2bitstore(i) + self._bitstore = helpers.ue2bitstore(i) - def _readue(self, pos: int) -> Tuple[int, int]: + def _readue(self, pos: int) -> tuple[int, int]: """Return interpretation of next bits as unsigned exponential-Golomb code. Raises ReadError if the end of the bitstring is encountered while reading the code. """ - if bitstring.options.lsb0: - raise bitstring.ReadError("Exp-Golomb codes cannot be read in lsb0 mode.") oldpos = pos try: while not self[pos]: @@ -855,25 +927,25 @@ def _readue(self, pos: int) -> Tuple[int, int]: pos += 1 return codenum, pos - def _getue(self) -> Tuple[int, int]: + def _getue(self) -> tuple[int, int]: try: return self._readue(0) except bitstring.ReadError: raise bitstring.InterpretError - def _getse(self) -> Tuple[int, int]: + def _getse(self) -> tuple[int, int]: try: return self._readse(0) except bitstring.ReadError: raise bitstring.InterpretError - def _getuie(self) -> Tuple[int, int]: + def _getuie(self) -> tuple[int, int]: try: return self._readuie(0) except bitstring.ReadError: raise bitstring.InterpretError - def _getsie(self) -> Tuple[int, int]: + def _getsie(self) -> tuple[int, int]: try: return self._readsie(0) except bitstring.ReadError: @@ -881,11 +953,9 @@ def _getsie(self) -> Tuple[int, int]: def _setse(self, i: int) -> None: """Initialise bitstring with signed exponential-Golomb code for integer i.""" - if bitstring.options.lsb0: - raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.") - self._bitstore = common_helpers.se2bitstore(i) + self._bitstore = helpers.se2bitstore(i) - def _readse(self, pos: int) -> Tuple[int, int]: + def _readse(self, pos: int) -> tuple[int, int]: """Return interpretation of next bits as a signed exponential-Golomb code. Advances position to after the read code. @@ -904,19 +974,15 @@ def _setuie(self, i: int) -> None: Raises CreationError if i < 0. """ - if bitstring.options.lsb0: - raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.") - self._bitstore = common_helpers.uie2bitstore(i) + self._bitstore = helpers.uie2bitstore(i) - def _readuie(self, pos: int) -> Tuple[int, int]: + def _readuie(self, pos: int) -> tuple[int, int]: """Return interpretation of next bits as unsigned interleaved exponential-Golomb code. Raises ReadError if the end of the bitstring is encountered while reading the code. """ - if bitstring.options.lsb0: - raise bitstring.ReadError("Exp-Golomb codes cannot be read in lsb0 mode.") try: codenum: int = 1 while not self[pos]: @@ -931,11 +997,9 @@ def _readuie(self, pos: int) -> Tuple[int, int]: def _setsie(self, i: int, ) -> None: """Initialise bitstring with signed interleaved exponential-Golomb code for integer i.""" - if bitstring.options.lsb0: - raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.") - self._bitstore = common_helpers.sie2bitstore(i) + self._bitstore = helpers.sie2bitstore(i) - def _readsie(self, pos: int) -> Tuple[int, int]: + def _readsie(self, pos: int) -> tuple[int, int]: """Return interpretation of next bits as a signed interleaved exponential-Golomb code. Advances position to after the read code. @@ -952,7 +1016,7 @@ def _readsie(self, pos: int) -> Tuple[int, int]: except IndexError: raise bitstring.ReadError("Read off end of bitstring trying to read code.") - def _setbool(self, value: Union[bool, str]) -> None: + def _setbool(self, value: bool | str) -> None: # We deliberately don't want to have implicit conversions to bool here. # If we did then it would be difficult to deal with the 'False' string. if value in (1, 'True', '1'): @@ -965,9 +1029,15 @@ def _setbool(self, value: Union[bool, str]) -> None: def _getbool(self) -> bool: return self[0] + def _readbool(self, pos: int, length: int) -> bool: + return bool(self._bitstore.getindex(pos)) + def _getpad(self) -> None: return None + def _readpad(self, pos: int, length: int) -> None: + return None + def _setpad(self, value: None, length: int) -> None: self._bitstore = ConstBitStore.from_zeros(length) @@ -979,6 +1049,9 @@ def _getbin(self) -> str: """Return interpretation as a binary string.""" return self._bitstore.to_bin() + def _readbin(self, pos: int, length: int) -> str: + return self._bitstore.read_bin(pos, length) + def _setoct(self, octstring: str, length: None = None) -> None: """Reset the bitstring to have the value given in octstring.""" self._bitstore = helpers.oct2bitstore(octstring) @@ -987,6 +1060,9 @@ def _getoct(self) -> str: """Return interpretation as an octal string.""" return self._bitstore.to_oct() + def _readoct(self, pos: int, length: int) -> str: + return self._bitstore.read_oct(pos, length) + def _sethex(self, hexstring: str, length: None = None) -> None: """Reset the bitstring to have the value given in hexstring.""" self._bitstore = helpers.hex2bitstore(hexstring) @@ -999,6 +1075,9 @@ def _gethex(self) -> str: """ return self._bitstore.to_hex() + def _readhex(self, pos: int, length: int) -> str: + return self._bitstore.read_hex(pos, length) + def _getlength(self) -> int: """Return the length of the bitstring in bits.""" return len(self._bitstore) @@ -1007,7 +1086,10 @@ def _copy(self: TBits) -> TBits: """Create and return a new copy of the Bits (always in memory).""" # Note that __copy__ may choose to return self if it's immutable. This method always makes a copy. s_copy = self.__class__() - s_copy._bitstore = self._bitstore._mutable_copy() + if isinstance(self._bitstore, ConstBitStore): + s_copy._bitstore = ConstBitStore(self._bitstore.tibs) + else: + s_copy._bitstore = self._bitstore._mutable_copy() return s_copy def _slice(self: TBits, start: int, end: int) -> TBits: @@ -1017,16 +1099,15 @@ def _slice(self: TBits, start: int, end: int) -> TBits: return bs def _absolute_slice(self: TBits, start: int, end: int) -> TBits: - """Used internally to get a slice, without error checking. - Uses MSB0 bit numbering even if LSB0 is set.""" + """Used internally to get a slice, without error checking.""" if end == start: return self.__class__() assert start < end, f"start={start}, end={end}" bs = self.__class__() - bs._bitstore = self._bitstore.getslice_msb0(start, end) + bs._bitstore = self._bitstore.getslice(start, end) return bs - def _readtoken(self, name: str, pos: int, length: Optional[int]) -> Tuple[Union[float, int, str, None, Bits], int]: + def _readtoken(self, name: str, pos: int, length: int | None) -> tuple[float | int | str | None | Bits, int]: """Reads a token from the bitstring and returns the result.""" dtype = dtype_register.get_dtype(name, length) if dtype.bitlength is not None and dtype.bitlength > len(self) - pos: @@ -1044,11 +1125,17 @@ def _readtoken(self, name: str, pos: int, length: Optional[int]) -> Tuple[Union[ def _addright(self, bs: Bits, /) -> None: """Add a bitstring to the RHS of the current bitstring.""" - self._bitstore += bs._bitstore + if isinstance(self._bitstore, ConstBitStore): + self._bitstore = self._bitstore + bs._bitstore + else: + self._bitstore += bs._bitstore def _addleft(self, bs: Bits, /) -> None: """Prepend a bitstring to the current bitstring.""" - self._bitstore.extend_left(bs._bitstore) + if isinstance(self._bitstore, ConstBitStore): + self._bitstore = bs._bitstore + self._bitstore + else: + self._bitstore.extend_left(bs._bitstore) def _insert(self, bs: Bits, pos: int, /) -> None: """Insert bs at pos.""" @@ -1074,7 +1161,7 @@ def _delete(self, bits: int, pos: int, /) -> None: def _reversebytes(self, start: int, end: int) -> None: """Reverse bytes in-place.""" assert (end - start) % 8 == 0 - self._bitstore[start:end] = ConstBitStore.from_bytes(self._bitstore.getslice(start, end).to_bytes()[::-1]) + self._bitstore.byte_swap(start, end) def _invert(self, pos: int, /) -> None: """Flip bit at pos 1<->0.""" @@ -1094,7 +1181,7 @@ def _irshift(self: TBits, n: int, /) -> TBits: def _getbits(self: TBits): return self._copy() - def _validate_slice(self, start: Optional[int], end: Optional[int]) -> Tuple[int, int]: + def _validate_slice(self, start: int | None, end: int | None) -> tuple[int, int]: """Validate start and end and return them as positive bit positions.""" start = 0 if start is None else (start + len(self) if start < 0 else start) end = len(self) if end is None else (end + len(self) if end < 0 else end) @@ -1102,7 +1189,7 @@ def _validate_slice(self, start: Optional[int], end: Optional[int]) -> Tuple[int raise ValueError(f"Invalid slice positions for bitstring length {len(self)}: start={start}, end={end}.") return start, end - def unpack(self, fmt: Union[str, List[Union[str, int]]], **kwargs) -> List[Union[int, float, str, Bits, bool, bytes, None]]: + def unpack(self, fmt: str | list[str | int], **kwargs) -> list[int | float | str | Bits | bool | bytes | None]: """Interpret the whole bitstring using fmt and return list. fmt -- A single string or a list of strings with comma separated tokens @@ -1119,8 +1206,8 @@ def unpack(self, fmt: Union[str, List[Union[str, int]]], **kwargs) -> List[Union """ return self._readlist(fmt, 0, **kwargs)[0] - def _readlist(self, fmt: Union[str, List[Union[str, int, Dtype]]], pos: int, **kwargs) \ - -> Tuple[List[Union[int, float, str, Bits, bool, bytes, None]], int]: + def _readlist(self, fmt: str | list[str | int | Dtype], pos: int, **kwargs) \ + -> tuple[list[int | float | str | Bits | bool | bytes | None], int]: if isinstance(fmt, str): fmt = [fmt] # Convert to a flat list of Dtypes @@ -1141,7 +1228,7 @@ def _readlist(self, fmt: Union[str, List[Union[str, int, Dtype]]], pos: int, **k dtype_list.append(Dtype(name, length)) return self._read_dtype_list(dtype_list, pos) - def _read_dtype_list(self, dtypes: List[Dtype], pos: int) -> Tuple[List[Union[int, float, str, Bits, bool, bytes, None]], int]: + def _read_dtype_list(self, dtypes: list[Dtype], pos: int) -> tuple[list[int | float | str | Bits | bool | bytes | None], int]: has_stretchy_token = False bits_after_stretchy_token = 0 for dtype in dtypes: @@ -1178,13 +1265,11 @@ def _read_dtype_list(self, dtypes: List[Dtype], pos: int) -> Tuple[List[Union[in vals.append(val) return vals, pos - def find(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None, - bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]: + def find(self, bs: BitsType, /, *, start: int | None = None, end: int | None = None, + bytealigned: bool | None = None) -> int | None: """Find first occurrence of substring bs. - Returns a single item tuple with the bit position if found, or an - empty tuple if not found. The bit position (pos property) will - also be set to the start of the substring if it is found. + Returns the bit position if found, or None if not found. bs -- The bitstring to find. start -- The bit position to start the search. Defaults to 0. @@ -1197,38 +1282,22 @@ def find(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] if end < start. >>> BitArray('0xc3e').find('0b1111') - (6,) + 6 """ bs = Bits._create_from_bitstype(bs) if len(bs) == 0: raise ValueError("Cannot find an empty bitstring.") - start, end = self._validate_slice(start, end) ba = bitstring.options.bytealigned if bytealigned is None else bytealigned p = self._find(bs, start, end, ba) return p - def _find_lsb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]: - # A forward find in lsb0 is very like a reverse find in msb0. - assert start <= end - assert bitstring.options.lsb0 - - new_slice = bitstring.helpers.offset_slice_indices_lsb0(slice(start, end, None), len(self)) - msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop) - p = self._rfind_msb0(bs, msb0_start, msb0_end, bytealigned) - - if p: - return (len(self) - p[0] - len(bs),) - else: - return () - - def _find_msb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]: + def _find(self, bs: Bits, start: int, end: int, bytealigned: bool) -> int | None: """Find first occurrence of a binary string.""" - p = self._bitstore.find(bs._bitstore, start, end, bytealigned) - return () if p is None else (p,) + return self._bitstore.find(bs._bitstore, start, end, bytealigned) - def findall(self, bs: BitsType, start: Optional[int] = None, end: Optional[int] = None, count: Optional[int] = None, - bytealigned: Optional[bool] = None) -> Iterable[int]: + def findall(self, bs: BitsType, *, start: int | None = None, end: int | None = None, count: int | None = None, + bytealigned: bool | None = None) -> Iterable[int]: """Find all occurrences of bs. Return generator of bit positions. bs -- The bitstring to find. @@ -1248,59 +1317,27 @@ def findall(self, bs: BitsType, start: Optional[int] = None, end: Optional[int] if count is not None and count < 0: raise ValueError("In findall, count must be >= 0.") bs = Bits._create_from_bitstype(bs) - start, end = self._validate_slice(start, end) ba = bitstring.options.bytealigned if bytealigned is None else bytealigned return self._findall(bs, start, end, count, ba) - def _findall_msb0(self, bs: Bits, start: int, end: int, count: Optional[int], + def _findall(self, bs: Bits, start: int, end: int, count: int | None, bytealigned: bool) -> Iterable[int]: - c = 0 - for i in self._bitstore.findall_msb0(bs._bitstore, start, end, bytealigned): - if count is not None and c >= count: + positions = self._bitstore.findall(bs._bitstore, start, end, bytealigned) + if count is None: + yield from positions + return + for _ in range(count): + try: + yield next(positions) + except StopIteration: return - c += 1 - yield i - return - def _findall_lsb0(self, bs: Bits, start: int, end: int, count: Optional[int], - bytealigned: bool) -> Iterable[int]: - assert start <= end - assert bitstring.options.lsb0 - new_slice = bitstring.helpers.offset_slice_indices_lsb0(slice(start, end, None), len(self)) - msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop) - - # Search chunks starting near the end and then moving back. - c = 0 - increment = max(8192, len(bs) * 80) - buffersize = min(increment + len(bs), msb0_end - msb0_start) - pos = max(msb0_start, msb0_end - buffersize) - while True: - found = list(self._findall_msb0(bs, start=pos, end=pos + buffersize, count=None, bytealigned=False)) - if not found: - if pos == msb0_start: - return - pos = max(msb0_start, pos - increment) - continue - while found: - if count is not None and c >= count: - return - c += 1 - lsb0_pos = len(self) - found.pop() - len(bs) - if not bytealigned or lsb0_pos % 8 == 0: - yield lsb0_pos - - pos = max(msb0_start, pos - increment) - if pos == msb0_start: - return - - def rfind(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None, - bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]: + def rfind(self, bs: BitsType, /, *, start: int | None = None, end: int | None = None, + bytealigned: bool | None = None) -> int | None: """Find final occurrence of substring bs. - Returns a single item tuple with the bit position if found, or an - empty tuple if not found. The bit position (pos property) will - also be set to the start of the substring if it is found. + Returns the bit position if found, or None if not found. bs -- The bitstring to find. start -- The bit position to end the reverse search. Defaults to 0. @@ -1314,33 +1351,16 @@ def rfind(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] """ bs = Bits._create_from_bitstype(bs) - start, end = self._validate_slice(start, end) ba = bitstring.options.bytealigned if bytealigned is None else bytealigned - if len(bs) == 0: - raise ValueError("Cannot find an empty bitstring.") p = self._rfind(bs, start, end, ba) return p - def _rfind_msb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]: + def _rfind(self, bs: Bits, start: int, end: int, bytealigned: bool) -> int | None: """Find final occurrence of a binary string.""" - p = self._bitstore.rfind(bs._bitstore, start, end, bytealigned) - return () if p is None else (p,) - - def _rfind_lsb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]: - # A reverse find in lsb0 is very like a forward find in msb0. - assert start <= end - assert bitstring.options.lsb0 - new_slice = bitstring.helpers.offset_slice_indices_lsb0(slice(start, end, None), len(self)) - msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop) - - p = self._find_msb0(bs, msb0_start, msb0_end, bytealigned) - if p: - return (len(self) - p[0] - len(bs),) - else: - return () + return self._bitstore.rfind(bs._bitstore, start, end, bytealigned) - def cut(self, bits: int, start: Optional[int] = None, end: Optional[int] = None, - count: Optional[int] = None) -> Iterator[Bits]: + def cut(self, bits: int, *, start: int | None = None, end: int | None = None, + count: int | None = None) -> Iterator[Bits]: """Return bitstring generator by cutting into bits sized chunks. bits -- The size in bits of the bitstring chunks to generate. @@ -1356,6 +1376,13 @@ def cut(self, bits: int, start: Optional[int] = None, end: Optional[int] = None, raise ValueError("Cannot cut - count must be >= 0.") if bits <= 0: raise ValueError("Cannot cut - bits must be >= 0.") + if isinstance(self._bitstore, ConstBitStore) and start_ == 0 and end_ == len(self): + cls = self.__class__ + for chunk_tibs in self._bitstore.tibs.chunks_iter(bits, count): + chunk = object.__new__(cls) + chunk._bitstore = ConstBitStore(chunk_tibs) + yield chunk + return c = 0 while count is None or c < count: c += 1 @@ -1368,8 +1395,8 @@ def cut(self, bits: int, start: Optional[int] = None, end: Optional[int] = None, start_ += bits return - def split(self, delimiter: BitsType, start: Optional[int] = None, end: Optional[int] = None, - count: Optional[int] = None, bytealigned: Optional[bool] = None) -> Iterable[Bits]: + def split(self, delimiter: BitsType, *, start: int | None = None, end: int | None = None, + count: int | None = None, bytealigned: bool | None = None) -> Iterable[Bits]: """Return bitstring generator by splitting using a delimiter. The first item returned is the initial bitstring before the delimiter, @@ -1386,6 +1413,7 @@ def split(self, delimiter: BitsType, start: Optional[int] = None, end: Optional[ Raises ValueError if the delimiter is empty. """ + # TODO: Delegate to Tibs.chunks_iter delimiter = Bits._create_from_bitstype(delimiter) if len(delimiter) == 0: raise ValueError("split delimiter cannot be empty.") @@ -1395,26 +1423,26 @@ def split(self, delimiter: BitsType, start: Optional[int] = None, end: Optional[ raise ValueError("Cannot split - count must be >= 0.") if count == 0: return - f = functools.partial(self._find_msb0, bs=delimiter, bytealigned=bytealigned_) + f = functools.partial(self._find, bs=delimiter, bytealigned=bytealigned_) found = f(start=start, end=end) - if not found: + if found is None: # Initial bits are the whole bitstring being searched yield self._slice(start, end) return # yield the bytes before the first occurrence of the delimiter, even if empty - yield self._slice(start, found[0]) - startpos = pos = found[0] + yield self._slice(start, found) + startpos = pos = found c = 1 while count is None or c < count: pos += len(delimiter) found = f(start=pos, end=end) - if not found: + if found is None: # No more occurrences, so return the rest of the bitstring yield self._slice(startpos, end) return c += 1 - yield self._slice(startpos, found[0]) - startpos = pos = found[0] + yield self._slice(startpos, found) + startpos = pos = found # Have generated count bitstrings, so time to quit. return @@ -1424,26 +1452,31 @@ def join(self: TBits, sequence: Iterable[Any]) -> TBits: sequence -- A sequence of bitstrings. """ - bs = MutableBitStore() - if len(self) == 0: - # Optimised version that doesn't need to add self between every item - for item in sequence: - bs += Bits._create_from_bitstype(item)._bitstore - else: + def _stores() -> Iterable[ConstBitStore | MutableBitStore]: + if len(self) == 0: + # Optimised version that doesn't need to add self between every item + for item in sequence: + yield Bits._create_from_bitstype(item)._bitstore + return + sequence_iter = iter(sequence) try: - bs += Bits._create_from_bitstype(next(sequence_iter))._bitstore + first = Bits._create_from_bitstype(next(sequence_iter))._bitstore except StopIteration: - pass - else: - for item in sequence_iter: - bs += self._bitstore - bs += Bits._create_from_bitstype(item)._bitstore + return + yield first + for item in sequence_iter: + yield self._bitstore + yield Bits._create_from_bitstype(item)._bitstore + s = self.__class__() - s._bitstore = bs + if isinstance(self._bitstore, ConstBitStore): + s._bitstore = ConstBitStore.join(_stores()) + else: + s._bitstore = MutableBitStore.join(_stores()) return s - def tobytes(self) -> bytes: + def to_bytes(self) -> bytes: """Return the bitstring as bytes, padding with zero bits if needed. Up to seven zero bits will be added at the end to byte align. @@ -1451,11 +1484,11 @@ def tobytes(self) -> bytes: """ return self._bitstore.to_bytes() - def tobitarray(self) -> bitarray.bitarray: - """Convert the bitstring to a bitarray object.""" - return self._bitstore.tobitarray() + def tobytes(self) -> bytes: + """Compatibility alias for :meth:`to_bytes`.""" + return self.to_bytes() - def tofile(self, f: BinaryIO) -> None: + def to_file(self, f: BinaryIO) -> None: """Write the bitstring to a file object, padding with zero bits if needed. Up to seven zero bits will be added at the end to byte align. @@ -1464,9 +1497,13 @@ def tofile(self, f: BinaryIO) -> None: # If the bitstring is file based then we don't want to read it all in to memory first. chunk_size = 8 * 100 * 1024 * 1024 # 100 MiB for chunk in self.cut(chunk_size): - f.write(chunk.tobytes()) + f.write(chunk.to_bytes()) - def startswith(self, prefix: BitsType, start: Optional[int] = None, end: Optional[int] = None) -> bool: + def tofile(self, f: BinaryIO) -> None: + """Compatibility alias for :meth:`to_file`.""" + self.to_file(f) + + def startswith(self, prefix: BitsType, *, start: int | None = None, end: int | None = None) -> bool: """Return whether the current bitstring starts with prefix. prefix -- The bitstring to search for. @@ -1478,7 +1515,7 @@ def startswith(self, prefix: BitsType, start: Optional[int] = None, end: Optiona start, end = self._validate_slice(start, end) return self._slice(start, start + len(prefix)) == prefix if end >= start + len(prefix) else False - def endswith(self, suffix: BitsType, start: Optional[int] = None, end: Optional[int] = None) -> bool: + def endswith(self, suffix: BitsType, *, start: int | None = None, end: int | None = None) -> bool: """Return whether the current bitstring ends with suffix. suffix -- The bitstring to search for. @@ -1490,7 +1527,7 @@ def endswith(self, suffix: BitsType, start: Optional[int] = None, end: Optional[ start, end = self._validate_slice(start, end) return self._slice(end - len(suffix), end) == suffix if start + len(suffix) <= end else False - def all(self, value: Any, pos: Optional[Iterable[int]] = None) -> bool: + def all(self, value: Any, pos: Iterable[int] | None = None) -> bool: """Return True if one or many bits are all set to bool(value). value -- If value is True then checks for bits set to 1, otherwise @@ -1507,7 +1544,7 @@ def all(self, value: Any, pos: Optional[Iterable[int]] = None) -> bool: return False return True - def any(self, value: Any, pos: Optional[Iterable[int]] = None) -> bool: + def any(self, value: Any, pos: Iterable[int] | None = None) -> bool: """Return True if any of one or many bits are set to bool(value). value -- If value is True then checks for bits set to 1, otherwise @@ -1534,23 +1571,21 @@ def count(self, value: Any) -> int: 7 """ - # count the number of 1s (from which it's easy to work out the 0s). - count = self._bitstore.count(1) - return count if value else len(self) - count + return self._bitstore.count(bool(value)) @staticmethod def _format_bits(bits: Bits, bits_per_group: int, sep: str, dtype: Dtype, - colour_start: str, colour_end: str, width: Optional[int]=None) -> Tuple[str, int]: + colour_start: str, colour_end: str, width: int | None=None) -> tuple[str, int]: get_fn = dtype.get_fn if dtype.name == 'bytes': # Special case for bytes to print one character each. get_fn = Bits._getbytes_printable if dtype.name == 'bool': # Special case for bool to print '1' or '0' instead of `True` or `False`. - get_fn = dtype_register.get_dtype('uint', bits_per_group).get_fn + get_fn = dtype_register.get_dtype('u', bits_per_group).get_fn if bits_per_group == 0: x = str(get_fn(bits)) else: - # Left-align for fixed width types when msb0, otherwise right-align. - align = '<' if dtype.name in ['bin', 'oct', 'hex', 'bits', 'bytes'] and not bitstring.options.lsb0 else '>' + # Left-align for fixed width types + align = '<' if dtype.name in ['bin', 'oct', 'hex', 'bits', 'bytes'] else '>' chars_per_group = 0 if dtype_register[dtype.name].bitlength2chars_fn is not None: chars_per_group = dtype_register[dtype.name].bitlength2chars_fn(bits_per_group) @@ -1560,14 +1595,11 @@ def _format_bits(bits: Bits, bits_per_group: int, sep: str, dtype: Dtype, padding_spaces = 0 if width is None else max(width - len(x), 0) x = colour_start + x + colour_end # Pad final line with spaces to align it - if bitstring.options.lsb0: - x = ' ' * padding_spaces + x - else: - x += ' ' * padding_spaces + x += ' ' * padding_spaces return x, chars_used @staticmethod - def _chars_per_group(bits_per_group: int, fmt: Optional[str]): + def _chars_per_group(bits_per_group: int, fmt: str | None): """How many characters are needed to represent a number of bits with a given format.""" if fmt is None or dtype_register[fmt].bitlength2chars_fn is None: return 0 @@ -1580,8 +1612,8 @@ def _bits_per_char(fmt: str): raise ValueError return 24 // dtype_register[fmt].bitlength2chars_fn(24) - def _pp(self, dtype1: Dtype, dtype2: Optional[Dtype], bits_per_group: int, width: int, sep: str, format_sep: str, - show_offset: bool, stream: TextIO, lsb0: bool, offset_factor: int) -> None: + def _pp(self, dtype1: Dtype, dtype2: Dtype | None, bits_per_group: int, width: int, sep: str, format_sep: str, + show_offset: bool, stream: TextIO, offset_factor: int) -> None: """Internal pretty print method.""" colour = Colour(not bitstring.options.no_color) name1 = dtype1.name @@ -1591,7 +1623,7 @@ def _pp(self, dtype1: Dtype, dtype2: Optional[Dtype], bits_per_group: int, width if dtype2 is not None and dtype2.variable_length: raise ValueError(f"Can't use Dtype '{dtype2}' in pp() as it has a variable length.") offset_width = 0 - offset_sep = ' :' if lsb0 else ': ' + offset_sep = ': ' if show_offset: # This could be 1 too large in some circumstances. Slightly recurrent logic needed to fix it... offset_width = len(str(len(self))) + len(offset_sep) @@ -1625,10 +1657,7 @@ def _pp(self, dtype1: Dtype, dtype2: Optional[Dtype], bits_per_group: int, width if show_offset: offset = bitpos // offset_factor bitpos += len(bits) - if bitstring.options.lsb0: - offset_str = colour.green + offset_sep + f'{offset: <{offset_width - len(offset_sep)}}' + colour.off - else: - offset_str = colour.green + f'{offset: >{offset_width - len(offset_sep)}}' + offset_sep + colour.off + offset_str = colour.green + f'{offset: >{offset_width - len(offset_sep)}}' + offset_sep + colour.off fb1, chars_used = Bits._format_bits(bits, bits_per_group, sep, dtype1, colour.purple, colour.off, first_fb_width) if first_fb_width is None: @@ -1641,10 +1670,7 @@ def _pp(self, dtype1: Dtype, dtype2: Optional[Dtype], bits_per_group: int, width second_fb_width = chars_used fb2 = format_sep + fb2 - if bitstring.options.lsb0 is True: - line_fmt = fb1 + fb2 + offset_str + '\n' - else: - line_fmt = offset_str + fb1 + fb2 + '\n' + line_fmt = offset_str + fb1 + fb2 + '\n' stream.write(line_fmt) return @@ -1683,7 +1709,7 @@ def _process_pp_tokens(token_list, fmt): bits_per_group //= 2 return dtype1, dtype2, bits_per_group, has_length_in_fmt - def pp(self, fmt: Optional[str] = None, width: int = 120, sep: str = ' ', + def pp(self, fmt: str | None = None, width: int = 120, sep: str = ' ', show_offset: bool = True, stream: TextIO = sys.stdout) -> None: """Pretty print the bitstring's value. @@ -1698,7 +1724,7 @@ def pp(self, fmt: Optional[str] = None, width: int = 120, sep: str = ' ', stream -- A TextIO object with a write() method. Defaults to sys.stdout. >>> s.pp('hex16') - >>> s.pp('b, h', sep='_', show_offset=False) + >>> s.pp('bin, hex', sep='_', show_offset=False) """ colour = Colour(not bitstring.options.no_color) @@ -1716,7 +1742,7 @@ def pp(self, fmt: Optional[str] = None, width: int = 120, sep: str = ' ', len_str = colour.green + str(len(self)) + colour.off output_stream.write(f"<{self.__class__.__name__}, fmt='{tidy_fmt}', length={len_str} bits> [\n") data._pp(dtype1, dtype2, bits_per_group, width, sep, format_sep, show_offset, - output_stream, bitstring.options.lsb0, 1) + output_stream, 1) output_stream.write("]") if trailing_bit_length != 0: output_stream.write(" + trailing_bits = " + str(self[-trailing_bit_length:])) @@ -1731,12 +1757,108 @@ def copy(self: TBits) -> TBits: return self @classmethod - def fromstring(cls: TBits, s: str, /) -> TBits: + def from_string(cls: type[TBits], s: str, /) -> TBits: """Create a new bitstring from a formatted string.""" x = super().__new__(cls) - x._bitstore = common_helpers.str_to_bitstore(s) + x._bitstore = helpers.str_to_bitstore(s) return x - len = length = property(_getlength, doc="The length of the bitstring in bits. Read only.") + @classmethod + def fromstring(cls: type[TBits], s: str, /) -> TBits: + """Compatibility alias for from_string().""" + return cls.from_string(s) + + @classmethod + def from_dtype(cls: type[TBits], dtype: str | Dtype, value: Any, /) -> TBits: + """Create a new bitstring by packing value according to dtype.""" + x = super().__new__(cls) + x._bitstore = Dtype(dtype).pack(value)._bitstore + return x + + @classmethod + def from_bytes(cls: type[TBits], data: bytes | bytearray | memoryview, /, *, + length: int | None = None, offset: int = 0) -> TBits: + """Create a new bitstring from a bytes-like object.""" + x = super().__new__(cls) + x._setbytes_with_truncation(data, length, offset) + return x + + @classmethod + def from_bools(cls: type[TBits], iterable: Iterable[Any], /) -> TBits: + """Create a new bitstring from an iterable of bool-like values.""" + x = super().__new__(cls) + x._bitstore = ConstBitStore.from_bools(iterable) + return x + + @classmethod + def from_zeros(cls: type[TBits], length: int, /) -> TBits: + """Create a new bitstring containing length zero bits.""" + length = int(length) + if length < 0: + raise bitstring.CreationError(f"Can't create bitstring of negative length {length}.") + x = super().__new__(cls) + x._bitstore = ConstBitStore.from_zeros(length) + return x + @classmethod + def from_ones(cls: type[TBits], length: int, /) -> TBits: + """Create a new bitstring containing length one bits.""" + length = int(length) + if length < 0: + raise bitstring.CreationError(f"Can't create bitstring of negative length {length}.") + x = super().__new__(cls) + x._bitstore = ConstBitStore.from_ones(length) + return x + @classmethod + def from_joined(cls: type[TBits], sequence: Iterable[BitsType], /) -> TBits: + """Create a new bitstring by concatenating a sequence of bitstrings.""" + sequence_iter = iter(sequence) + x = super().__new__(cls) + try: + first = Bits._create_from_bitstype(next(sequence_iter))._bitstore + except StopIteration: + x._bitstore = ConstBitStore.from_zeros(0) + return x + + def stores() -> Iterable[ConstBitStore | MutableBitStore]: + yield first + for item in sequence_iter: + yield Bits._create_from_bitstype(item)._bitstore + + x._bitstore = ConstBitStore.join(stores()) + return x + + @classmethod + def from_file(cls: type[TBits], source: str | pathlib.Path | BinaryIO, /, *, + length: int | None = None, offset: int = 0) -> TBits: + """Create a new bitstring from a file path or binary file object.""" + x = super().__new__(cls) + filename = source if isinstance(source, (str, pathlib.Path)) else source.name + x._setfile(filename, length, offset) + return x + + @classmethod + def from_tibs(cls: type[TBits], tibs: Tibs | Mutibs, /) -> TBits: + """Create a new bitstring from a tibs.Tibs or tibs.Mutibs instance.""" + if isinstance(tibs, Mutibs): + tibs = tibs.to_tibs() + elif not isinstance(tibs, Tibs): + raise TypeError(f"Expected tibs.Tibs or tibs.Mutibs, got {type(tibs).__name__}.") + x = super().__new__(cls) + x._bitstore = ConstBitStore(tibs) + return x + + def to_bitarray(self) -> bitstring.BitArray: + """Return a mutable copy of the bitstring.""" + from bitstring.bitarray_ import BitArray + + x = BitArray() + x._bitstore = self._bitstore._mutable_copy() + return x + + def to_tibs(self) -> Tibs: + """Return the data as a tibs.Tibs instance.""" + return self._bitstore.tibs + + len = length = property(_getlength, doc="The length of the bitstring in bits. Read only.") diff --git a/bitstring/bitstore_bitarray.py b/bitstring/bitstore_bitarray.py deleted file mode 100644 index 80e9d1a4..00000000 --- a/bitstring/bitstore_bitarray.py +++ /dev/null @@ -1,328 +0,0 @@ -from __future__ import annotations - -import bitarray -import bitarray.util -from bitstring.exceptions import CreationError -from typing import Union, Iterable, Optional, overload, Iterator, Any -from bitstring.helpers import offset_slice_indices_lsb0 - -if bitarray.__version__.startswith("2."): - raise ImportError(f"bitstring version 4.3 requires bitarray version 3 or higher. Found version {bitarray.__version__}.") - - -class _BitStore: - """A light wrapper around bitarray that does the LSB0 stuff""" - - __slots__ = ('_bitarray', 'modified_length', 'immutable') - - def __init__(self, initializer: Union[bitarray.bitarray, None] = None, - immutable: bool = False) -> None: - if isinstance(initializer, str): - assert False - self._bitarray = bitarray.bitarray(initializer) - self.immutable = immutable - self.modified_length = None - - @classmethod - def from_zeros(cls, i: int) -> _BitStore: - x = super().__new__(cls) - x._bitarray = bitarray.bitarray(i) - x.immutable = False - x.modified_length = None - return x - - - @classmethod - def from_bin(cls, s: str) -> _BitStore: - x = super().__new__(cls) - x._bitarray = bitarray.bitarray(s) - x.immutable = False - x.modified_length = None - return x - - @classmethod - def from_bytes(cls, b: Union[bytes, bytearray, memoryview], /) -> _BitStore: - x = super().__new__(cls) - x._bitarray = bitarray.bitarray() - x._bitarray.frombytes(b) - x.immutable = False - x.modified_length = None - return x - - @classmethod - def frombuffer(cls, buffer, /, length: Optional[int] = None) -> _BitStore: - x = super().__new__(cls) - x._bitarray = bitarray.bitarray(buffer=buffer) - x.immutable = True - x.modified_length = length - # Here 'modified' means it shouldn't be changed further, so setting, deleting etc. are disallowed. - if x.modified_length is not None: - if x.modified_length < 0: - raise CreationError("Can't create bitstring with a negative length.") - if x.modified_length > len(x._bitarray): - raise CreationError( - f"Can't create bitstring with a length of {x.modified_length} from {len(x._bitarray)} bits of data.") - return x - - @classmethod - def join(cls, bitstores: Iterable[_BitStore], /) -> _BitStore: - x = super().__new__(cls) - x._bitarray = bitarray.bitarray() - for b in bitstores: - x._bitarray += b._bitarray - x.immutable = False - x.modified_length = None - return x - - @staticmethod - def using_rust_core() -> bool: - return False - - def tobitarray(self) -> bitarray.bitarray: - if self.modified_length is not None: - return self.getslice(0, len(self))._bitarray - return self._bitarray - - def to_bytes(self) -> bytes: - if self.modified_length is not None: - return self._bitarray[:self.modified_length].tobytes() - return self._bitarray.tobytes() - - def to_u(self) -> int: - if self.modified_length is not None: - return bitarray.util.ba2int(self._bitarray[:self.modified_length], signed=False) - return bitarray.util.ba2int(self._bitarray, signed=False) - - def to_i(self) -> int: - if self.modified_length is not None: - return bitarray.util.ba2int(self._bitarray[:self.modified_length], signed=True) - return bitarray.util.ba2int(self._bitarray, signed=True) - - def to_hex(self) -> str: - if self.modified_length is not None: - return bitarray.util.ba2hex(self._bitarray[:self.modified_length]) - return bitarray.util.ba2hex(self._bitarray) - - def to_bin(self) -> str: - if self.modified_length is not None: - return self._bitarray[:self.modified_length].to01() - return self._bitarray.to01() - - def to_oct(self) -> str: - if self.modified_length is not None: - return bitarray.util.ba2base(8, self._bitarray[:self.modified_length]) - return bitarray.util.ba2base(8, self._bitarray) - - def __imul__(self, n: int, /) -> _BitStore: - self._bitarray *= n - return self - - def __ilshift__(self, n: int, /) -> None: - self._bitarray <<= n - - def __irshift__(self, n: int, /) -> None: - self._bitarray >>= n - - def __iadd__(self, other: _BitStore, /) -> _BitStore: - self._bitarray += other._bitarray - return self - - def __add__(self, other: _BitStore, /) -> _BitStore: - bs = self._mutable_copy() - bs += other - return bs - - def __eq__(self, other: Any, /) -> bool: - return self._bitarray == other._bitarray - - def __and__(self, other: _BitStore, /) -> _BitStore: - return _BitStore(self._bitarray & other._bitarray) - - def __or__(self, other: _BitStore, /) -> _BitStore: - return _BitStore(self._bitarray | other._bitarray) - - def __xor__(self, other: _BitStore, /) -> _BitStore: - return _BitStore(self._bitarray ^ other._bitarray) - - def __iand__(self, other: _BitStore, /) -> _BitStore: - self._bitarray &= other._bitarray - return self - - def __ior__(self, other: _BitStore, /) -> _BitStore: - self._bitarray |= other._bitarray - return self - - def __ixor__(self, other: _BitStore, /) -> _BitStore: - self._bitarray ^= other._bitarray - return self - - def __invert__(self) -> _BitStore: - return _BitStore(~self._bitarray) - - def find(self, bs: _BitStore, start: int, end: int, bytealigned: bool = False) -> int | None: - if not bytealigned: - x = self._bitarray.find(bs._bitarray, start, end) - return None if x == -1 else x - try: - return next(self.findall_msb0(bs, start, end, bytealigned)) - except StopIteration: - return None - - def rfind(self, bs: _BitStore, start: int, end: int, bytealigned: bool = False) -> int | None: - if not bytealigned: - x = self._bitarray.find(bs._bitarray, start, end, right=True) - return None if x == -1 else x - try: - return next(self.rfindall_msb0(bs, start, end, bytealigned)) - except StopIteration: - return None - - def findall_msb0(self, bs: _BitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: - if bytealigned is True and len(bs) % 8 == 0: - # Special case, looking for whole bytes on whole byte boundaries - bytes_ = bs.to_bytes() - # Round up start byte to next byte, and round end byte down. - # We're only looking for whole bytes, so can ignore bits at either end. - start_byte = (start + 7) // 8 - end_byte = end // 8 - b = self._bitarray[start_byte * 8: end_byte * 8].tobytes() - byte_pos = 0 - bytes_to_search = end_byte - start_byte - while byte_pos < bytes_to_search: - byte_pos = b.find(bytes_, byte_pos) - if byte_pos == -1: - break - yield (byte_pos + start_byte) * 8 - byte_pos = byte_pos + 1 - return - # General case - i = self._bitarray.search(bs._bitarray, start, end) - if not bytealigned: - for p in i: - yield p - else: - for p in i: - if (p % 8) == 0: - yield p - - def rfindall_msb0(self, bs: _BitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: - i = self._bitarray.search(bs._bitarray, start, end, right=True) - if not bytealigned: - for p in i: - yield p - else: - for p in i: - if (p % 8) == 0: - yield p - - def count(self, value, /) -> int: - return self._bitarray.count(value) - - def clear(self) -> None: - self._bitarray.clear() - - def reverse(self) -> None: - self._bitarray.reverse() - - def __iter__(self) -> Iterable[bool]: - for i in range(len(self)): - yield self.getindex(i) - - def _mutable_copy(self) -> _BitStore: - """Always creates a copy, even if instance is immutable.""" - return _BitStore(self._bitarray, immutable=False) - - def as_immutable(self) -> _BitStore: - return _BitStore(self._bitarray, immutable=True) - - def copy(self) -> _BitStore: - return self if self.immutable else self._mutable_copy() - - def __getitem__(self, item: Union[int, slice], /) -> Union[int, _BitStore]: - # Use getindex or getslice instead - raise NotImplementedError - - def getindex_msb0(self, index: int, /) -> bool: - return bool(self._bitarray.__getitem__(index)) - - def getslice_withstep_msb0(self, key: slice, /) -> _BitStore: - if self.modified_length is not None: - key = slice(*key.indices(self.modified_length)) - return _BitStore(self._bitarray.__getitem__(key)) - - def getslice_withstep_lsb0(self, key: slice, /) -> _BitStore: - key = offset_slice_indices_lsb0(key, len(self)) - return _BitStore(self._bitarray.__getitem__(key)) - - def getslice_msb0(self, start: Optional[int], stop: Optional[int], /) -> _BitStore: - if self.modified_length is not None: - key = slice(*slice(start, stop, None).indices(self.modified_length)) - start = key.start - stop = key.stop - return _BitStore(self._bitarray[start:stop]) - - def getslice_lsb0(self, start: Optional[int], stop: Optional[int], /) -> _BitStore: - s = offset_slice_indices_lsb0(slice(start, stop, None), len(self)) - return _BitStore(self._bitarray[s.start:s.stop]) - - def getindex_lsb0(self, index: int, /) -> bool: - return bool(self._bitarray.__getitem__(-index - 1)) - - @overload - def setitem_lsb0(self, key: int, value: int, /) -> None: - ... - - @overload - def setitem_lsb0(self, key: slice, value: _BitStore, /) -> None: - ... - - def setitem_lsb0(self, key: Union[int, slice], value: Union[int, _BitStore], /) -> None: - if isinstance(key, slice): - new_slice = offset_slice_indices_lsb0(key, len(self)) - self._bitarray.__setitem__(new_slice, value._bitarray) - else: - self._bitarray.__setitem__(-key - 1, value) - - def delitem_lsb0(self, key: Union[int, slice], /) -> None: - if isinstance(key, slice): - new_slice = offset_slice_indices_lsb0(key, len(self)) - self._bitarray.__delitem__(new_slice) - else: - self._bitarray.__delitem__(-key - 1) - - def invert_msb0(self, index: Optional[int] = None, /) -> None: - if index is not None: - self._bitarray.invert(index) - else: - self._bitarray.invert() - - def invert_lsb0(self, index: Optional[int] = None, /) -> None: - if index is not None: - self._bitarray.invert(-index - 1) - else: - self._bitarray.invert() - - def extend_left(self, other: _BitStore, /) -> None: - self._bitarray = other._bitarray + self._bitarray - - def any(self) -> bool: - return self._bitarray.any() - - def all(self) -> bool: - return self._bitarray.all() - - def __len__(self) -> int: - return self.modified_length if self.modified_length is not None else len(self._bitarray) - - def setitem_msb0(self, key, value, /): - if isinstance(value, _BitStore): - self._bitarray.__setitem__(key, value._bitarray) - else: - self._bitarray.__setitem__(key, value) - - def delitem_msb0(self, key, /): - self._bitarray.__delitem__(key) - - -ConstBitStore = _BitStore -MutableBitStore = _BitStore \ No newline at end of file diff --git a/bitstring/bitstore_bitarray_helpers.py b/bitstring/bitstore_bitarray_helpers.py deleted file mode 100644 index 336957df..00000000 --- a/bitstring/bitstore_bitarray_helpers.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -import struct -import math -import functools -from typing import Union, Optional, Dict, Callable -import bitarray -import bitstring -from bitstring.fp8 import p4binary_fmt, p3binary_fmt -from bitstring.mxfp import (e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt, e4m3mxfp_saturate_fmt, - e5m2mxfp_saturate_fmt, e4m3mxfp_overflow_fmt, e5m2mxfp_overflow_fmt) -from bitstring.helpers import tidy_input_string - -ConstBitStore = bitstring.bitstore.ConstBitStore -MutableBitStore = bitstring.bitstore.MutableBitStore - - -def bin2bitstore(binstring: str) -> ConstBitStore: - binstring = tidy_input_string(binstring) - binstring = binstring.replace('0b', '') - try: - return ConstBitStore.from_bin(binstring) - except ValueError: - raise bitstring.CreationError(f"Invalid character in bin initialiser {binstring}.") - - -def hex2bitstore(hexstring: str) -> ConstBitStore: - hexstring = tidy_input_string(hexstring) - hexstring = hexstring.replace('0x', '') - try: - ba = bitarray.util.hex2ba(hexstring) - except ValueError: - raise bitstring.CreationError("Invalid symbol in hex initialiser.") - return ConstBitStore(ba) - - -def oct2bitstore(octstring: str) -> ConstBitStore: - octstring = tidy_input_string(octstring) - octstring = octstring.replace('0o', '') - try: - ba = bitarray.util.base2ba(8, octstring) - except ValueError: - raise bitstring.CreationError("Invalid symbol in oct initialiser.") - return ConstBitStore(ba) - - -def int2bitstore(i: int, length: int, signed: bool) -> ConstBitStore: - i = int(i) - try: - x = ConstBitStore(bitarray.util.int2ba(i, length=length, endian='big', signed=signed)) - except OverflowError as e: - if signed: - if i >= (1 << (length - 1)) or i < -(1 << (length - 1)): - raise bitstring.CreationError(f"{i} is too large a signed integer for a bitstring of length {length}. " - f"The allowed range is [{-(1 << (length - 1))}, {(1 << (length - 1)) - 1}].") - else: - if i >= (1 << length): - raise bitstring.CreationError(f"{i} is too large an unsigned integer for a bitstring of length {length}. " - f"The allowed range is [0, {(1 << length) - 1}].") - if i < 0: - raise bitstring.CreationError("uint cannot be initialised with a negative number.") - raise e - return x - - -def intle2bitstore(i: int, length: int, signed: bool) -> ConstBitStore: - x = int2bitstore(i, length, signed).to_bytes() - return ConstBitStore.from_bytes(x[::-1]) - - -def float2bitstore(f: Union[str, float], length: int, big_endian: bool) -> ConstBitStore: - f = float(f) - fmt = {16: '>e', 32: '>f', 64: '>d'}[length] if big_endian else {16: ' 0 else float('-inf')) - return ConstBitStore.from_bytes(b) - diff --git a/bitstring/bitstore_common_helpers.py b/bitstring/bitstore_common_helpers.py deleted file mode 100644 index 5bc409e2..00000000 --- a/bitstring/bitstore_common_helpers.py +++ /dev/null @@ -1,192 +0,0 @@ -from __future__ import annotations - -import struct -import math -from typing import Union, Dict, Callable, Optional -import functools -import bitstring -from bitstring.fp8 import p4binary_fmt, p3binary_fmt -from bitstring.mxfp import (e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt, e4m3mxfp_saturate_fmt, - e5m2mxfp_saturate_fmt, e4m3mxfp_overflow_fmt, e5m2mxfp_overflow_fmt) - -helpers = bitstring.bitstore_helpers -ConstBitStore = bitstring.bitstore.ConstBitStore -MutableBitStore = bitstring.bitstore.MutableBitStore - - -CACHE_SIZE = 256 - -@functools.lru_cache(CACHE_SIZE) -def str_to_bitstore(s: str) -> ConstBitStore: - _, tokens = bitstring.utils.tokenparser(s) - constbitstores = [bitstore_from_token(*token) for token in tokens] - return ConstBitStore.join(constbitstores) - - -literal_bit_funcs: Dict[str, Callable[..., ConstBitStore]] = { - '0x': helpers.hex2bitstore, - '0X': helpers.hex2bitstore, - '0b': helpers.bin2bitstore, - '0B': helpers.bin2bitstore, - '0o': helpers.oct2bitstore, - '0O': helpers.oct2bitstore, -} - - -def bitstore_from_token(name: str, token_length: Optional[int], value: Optional[str]) -> ConstBitStore: - if name in literal_bit_funcs: - return literal_bit_funcs[name](value) - try: - d = bitstring.dtypes.Dtype(name, token_length) - except ValueError as e: - raise bitstring.CreationError(f"Can't parse token: {e}") - if value is None and name != 'pad': - raise ValueError(f"Token {name} requires a value.") - bs = d.build(value)._bitstore - if token_length is not None and len(bs) != d.bitlength: - raise bitstring.CreationError(f"Token with length {token_length} packed with value of length {len(bs)} " - f"({name}:{token_length}={value}).") - return bs - - - -def ue2bitstore(i: Union[str, int]) -> ConstBitStore: - i = int(i) - if i < 0: - raise bitstring.CreationError("Cannot use negative initialiser for unsigned exponential-Golomb.") - if i == 0: - return ConstBitStore.from_bin('1') - tmp = i + 1 - leadingzeros = -1 - while tmp > 0: - tmp >>= 1 - leadingzeros += 1 - remainingpart = i + 1 - (1 << leadingzeros) - return ConstBitStore.from_bin('0' * leadingzeros + '1') + helpers.int2bitstore(remainingpart, leadingzeros, False) - - -def se2bitstore(i: Union[str, int]) -> ConstBitStore: - i = int(i) - if i > 0: - u = (i * 2) - 1 - else: - u = -2 * i - return ue2bitstore(u) - - -def uie2bitstore(i: Union[str, int]) -> ConstBitStore: - i = int(i) - if i < 0: - raise bitstring.CreationError("Cannot use negative initialiser for unsigned interleaved exponential-Golomb.") - return ConstBitStore.from_bin('1' if i == 0 else '0' + '0'.join(bin(i + 1)[3:]) + '1') - - -def sie2bitstore(i: Union[str, int]) -> ConstBitStore: - i = int(i) - if i == 0: - return ConstBitStore.from_bin('1') - else: - return uie2bitstore(abs(i)) + (ConstBitStore.from_bin('1') if i < 0 else ConstBitStore.from_bin('0')) - - -def bfloat2bitstore(f: Union[str, float], big_endian: bool) -> ConstBitStore: - f = float(f) - fmt = '>f' if big_endian else ' 0 else float('-inf')) - return ConstBitStore.from_bytes(b[0:2]) if big_endian else ConstBitStore.from_bytes(b[2:4]) - - -def p4binary2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - u = p4binary_fmt.float_to_int8(f) - return helpers.int2bitstore(u, 8, False) - - -def p3binary2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - u = p3binary_fmt.float_to_int8(f) - return helpers.int2bitstore(u, 8, False) - - -def e4m3mxfp2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - if bitstring.options.mxfp_overflow == 'saturate': - u = e4m3mxfp_saturate_fmt.float_to_int(f) - else: - u = e4m3mxfp_overflow_fmt.float_to_int(f) - return helpers.int2bitstore(u, 8, False) - - -def e5m2mxfp2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - if bitstring.options.mxfp_overflow == 'saturate': - u = e5m2mxfp_saturate_fmt.float_to_int(f) - else: - u = e5m2mxfp_overflow_fmt.float_to_int(f) - return helpers.int2bitstore(u, 8, False) - - -def e3m2mxfp2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - if math.isnan(f): - raise ValueError("Cannot convert float('nan') to e3m2mxfp format as it has no representation for it.") - u = e3m2mxfp_fmt.float_to_int(f) - return helpers.int2bitstore(u, 6, False) - - -def e2m3mxfp2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - if math.isnan(f): - raise ValueError("Cannot convert float('nan') to e2m3mxfp format as it has no representation for it.") - u = e2m3mxfp_fmt.float_to_int(f) - return helpers.int2bitstore(u, 6, False) - - -def e2m1mxfp2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - if math.isnan(f): - raise ValueError("Cannot convert float('nan') to e2m1mxfp format as it has no representation for it.") - u = e2m1mxfp_fmt.float_to_int(f) - return helpers.int2bitstore(u, 4, False) - - -e8m0mxfp_allowed_values = [float(2 ** x) for x in range(-127, 128)] - - -def e8m0mxfp2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - if math.isnan(f): - return ConstBitStore.from_bin('11111111') - try: - i = e8m0mxfp_allowed_values.index(f) - except ValueError: - raise ValueError(f"{f} is not a valid e8m0mxfp value. It must be exactly 2 ** i, for -127 <= i <= 127 or float('nan') as no rounding will be done.") - return helpers.int2bitstore(i, 8, False) - - -def mxint2bitstore(f: Union[str, float]) -> ConstBitStore: - f = float(f) - if math.isnan(f): - raise ValueError("Cannot convert float('nan') to mxint format as it has no representation for it.") - f *= 2 ** 6 # Remove the implicit scaling factor - if f > 127: # 1 + 63/64 - return ConstBitStore.from_bin('01111111') - if f <= -128: # -2 - return ConstBitStore.from_bin('10000000') - # Want to round to nearest, so move by 0.5 away from zero and round down by converting to int - if f >= 0.0: - f += 0.5 - i = int(f) - # For ties-round-to-even - if f - i == 0.0 and i % 2: - i -= 1 - else: - f -= 0.5 - i = int(f) - if f - i == 0.0 and i % 2: - i += 1 - return helpers.int2bitstore(i, 8, True) \ No newline at end of file diff --git a/bitstring/bitstore_helpers.py b/bitstring/bitstore_helpers.py new file mode 100644 index 00000000..db44a7b7 --- /dev/null +++ b/bitstring/bitstore_helpers.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +from tibs import Tibs, ByteOrder + +import struct +import math +from collections.abc import Callable +import functools +import bitstring +from bitstring.fp8 import p4binary_fmt, p3binary_fmt +from bitstring.mxfp import (e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt, e4m3mxfp_saturate_fmt, + e5m2mxfp_saturate_fmt, e4m3mxfp_overflow_fmt, e5m2mxfp_overflow_fmt) + + +MutableBitStore = bitstring.bitstore.MutableBitStore +ConstBitStore = bitstring.bitstore.ConstBitStore + +from bitstring.helpers import tidy_input_string + + +def _int_to_tibs(i: int, length: int, signed: bool, little_endian: bool) -> Tibs: + try: + if signed: + return Tibs.from_i(i, length, ByteOrder.Little if little_endian else ByteOrder.Unspecified) + return Tibs.from_u(i, length, ByteOrder.Little if little_endian else ByteOrder.Unspecified) + except (OverflowError, ValueError) as e: + # Keep tibs validation for normal sizes and unsupported values. + if length <= 128: + raise ValueError(e) + byteorder = "little" if little_endian else "big" + b = i.to_bytes((length + 7) // 8, byteorder=byteorder, signed=signed) + offset = (-length) % 8 + return Tibs.from_bytes(b, offset=offset, length=length) + + +def bin2bitstore(binstring: str) -> ConstBitStore: + binstring = tidy_input_string(binstring) + binstring = binstring.replace('0b', '') + return ConstBitStore.from_bin(binstring) + + +def hex2bitstore(hexstring: str) -> ConstBitStore: + hexstring = tidy_input_string(hexstring) + hexstring = hexstring.replace('0x', '') + return ConstBitStore(Tibs.from_hex(hexstring)) + + +def oct2bitstore(octstring: str) -> ConstBitStore: + octstring = tidy_input_string(octstring) + octstring = octstring.replace('0o', '') + return ConstBitStore(Tibs.from_oct(octstring)) + + +def int2bitstore(i: int, length: int, signed: bool) -> ConstBitStore: + i = int(i) + t = _int_to_tibs(i, length, signed, little_endian=False) + return ConstBitStore(t) + + +def intle2bitstore(i: int, length: int, signed: bool) -> ConstBitStore: + i = int(i) + t = _int_to_tibs(i, length, signed, little_endian=True) + return ConstBitStore(t) + + +def float2bitstore(f: str | float, length: int, big_endian: bool) -> ConstBitStore: + f = float(f) + t = Tibs.from_f(f, length, ByteOrder.Big if big_endian else ByteOrder.Little) + return ConstBitStore(t) + + +CACHE_SIZE = 256 + +def _to_const_bitstore(bs: MutableBitStore | ConstBitStore) -> ConstBitStore: + if isinstance(bs, ConstBitStore): + return bs + return ConstBitStore(bs.tibs.to_tibs()) + + +def _bin_literal_to_const_bitstore(binstring: str) -> ConstBitStore: + binstring = tidy_input_string(binstring) + binstring = binstring.replace('0b', '') + return ConstBitStore.from_bin(binstring) + + +def _hex_literal_to_const_bitstore(hexstring: str) -> ConstBitStore: + hexstring = tidy_input_string(hexstring) + hexstring = hexstring.replace('0x', '') + return ConstBitStore(Tibs.from_hex(hexstring)) + + +def _oct_literal_to_const_bitstore(octstring: str) -> ConstBitStore: + octstring = tidy_input_string(octstring) + octstring = octstring.replace('0o', '') + return ConstBitStore(Tibs.from_oct(octstring)) + + +@functools.lru_cache(CACHE_SIZE) +def str_to_bitstore(s: str) -> ConstBitStore: + # Fast path for literal-only strings (e.g. "0xff, 0b101, 0o7"). + try: + return ConstBitStore(Tibs.from_string(s)) + except ValueError: + pass + _, tokens = bitstring.utils.tokenparser(s) + constbitstores = [bitstore_from_token(*token) for token in tokens] + return ConstBitStore.join(constbitstores) + + +literal_bit_funcs: dict[str, Callable[..., ConstBitStore]] = { + '0x': _hex_literal_to_const_bitstore, + '0X': _hex_literal_to_const_bitstore, + '0b': _bin_literal_to_const_bitstore, + '0B': _bin_literal_to_const_bitstore, + '0o': _oct_literal_to_const_bitstore, + '0O': _oct_literal_to_const_bitstore, +} + + +def bitstore_from_token(name: str, token_length: int | None, value: str | None) -> ConstBitStore: + if name in literal_bit_funcs: + return literal_bit_funcs[name](value) + try: + d = bitstring.dtypes.Dtype(name, token_length) + except ValueError as e: + raise bitstring.CreationError(f"Can't parse token: {e}") + if value is None and name != 'pad': + raise ValueError(f"Token {name} requires a value.") + bs = d.pack(value)._bitstore + bs = _to_const_bitstore(bs) + if token_length is not None and len(bs) != d.bitlength: + raise bitstring.CreationError(f"Token with length {token_length} packed with value of length {len(bs)} " + f"({name}:{token_length}={value}).") + return bs + + + +def ue2bitstore(i: str | int) -> ConstBitStore: + i = int(i) + if i < 0: + raise bitstring.CreationError("Cannot use negative initialiser for unsigned exponential-Golomb.") + if i == 0: + return ConstBitStore.from_bin('1') + tmp = i + 1 + leadingzeros = -1 + while tmp > 0: + tmp >>= 1 + leadingzeros += 1 + remainingpart = i + 1 - (1 << leadingzeros) + return ConstBitStore.from_bin('0' * leadingzeros + '1') + int2bitstore(remainingpart, leadingzeros, False) + + +def se2bitstore(i: str | int) -> ConstBitStore: + i = int(i) + if i > 0: + u = (i * 2) - 1 + else: + u = -2 * i + return ue2bitstore(u) + + +def uie2bitstore(i: str | int) -> ConstBitStore: + i = int(i) + if i < 0: + raise bitstring.CreationError("Cannot use negative initialiser for unsigned interleaved exponential-Golomb.") + return ConstBitStore.from_bin('1' if i == 0 else '0' + '0'.join(bin(i + 1)[3:]) + '1') + + +def sie2bitstore(i: str | int) -> ConstBitStore: + i = int(i) + if i == 0: + return ConstBitStore.from_bin('1') + else: + return uie2bitstore(abs(i)) + (ConstBitStore.from_bin('1') if i < 0 else ConstBitStore.from_bin('0')) + + +def bfloat2bitstore(f: str | float, big_endian: bool) -> ConstBitStore: + f = float(f) + fmt = '>f' if big_endian else ' 0 else float('-inf')) + return ConstBitStore.from_bytes(b[0:2]) if big_endian else ConstBitStore.from_bytes(b[2:4]) + + +def p4binary2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + u = p4binary_fmt.float_to_int8(f) + return int2bitstore(u, 8, False) + + +def p3binary2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + u = p3binary_fmt.float_to_int8(f) + return int2bitstore(u, 8, False) + + +def e4m3mxfp_saturate2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + u = e4m3mxfp_saturate_fmt.float_to_int(f) + return int2bitstore(u, 8, False) + + +def e4m3mxfp_overflow2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + u = e4m3mxfp_overflow_fmt.float_to_int(f) + return int2bitstore(u, 8, False) + + +def e5m2mxfp_saturate2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + u = e5m2mxfp_saturate_fmt.float_to_int(f) + return int2bitstore(u, 8, False) + + +def e5m2mxfp_overflow2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + u = e5m2mxfp_overflow_fmt.float_to_int(f) + return int2bitstore(u, 8, False) + + +def e3m2mxfp2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + if math.isnan(f): + raise ValueError("Cannot convert float('nan') to e3m2mxfp format as it has no representation for it.") + u = e3m2mxfp_fmt.float_to_int(f) + return int2bitstore(u, 6, False) + + +def e2m3mxfp2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + if math.isnan(f): + raise ValueError("Cannot convert float('nan') to e2m3mxfp format as it has no representation for it.") + u = e2m3mxfp_fmt.float_to_int(f) + return int2bitstore(u, 6, False) + + +def e2m1mxfp2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + if math.isnan(f): + raise ValueError("Cannot convert float('nan') to e2m1mxfp format as it has no representation for it.") + u = e2m1mxfp_fmt.float_to_int(f) + return int2bitstore(u, 4, False) + + +e8m0mxfp_allowed_values = [float(2 ** x) for x in range(-127, 128)] + + +def e8m0mxfp2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + if math.isnan(f): + return ConstBitStore.from_bin('11111111') + try: + i = e8m0mxfp_allowed_values.index(f) + except ValueError: + raise ValueError(f"{f} is not a valid e8m0mxfp value. It must be exactly 2 ** i, for -127 <= i <= 127 or float('nan') as no rounding will be done.") + return int2bitstore(i, 8, False) + + +def mxint2bitstore(f: str | float) -> ConstBitStore: + f = float(f) + if math.isnan(f): + raise ValueError("Cannot convert float('nan') to mxint format as it has no representation for it.") + f *= 2 ** 6 # Remove the implicit scaling factor + if f > 127: # 1 + 63/64 + return ConstBitStore.from_bin('01111111') + if f <= -128: # -2 + return ConstBitStore.from_bin('10000000') + # Want to round to nearest, so move by 0.5 away from zero and round down by converting to int + if f >= 0.0: + f += 0.5 + i = int(f) + # For ties-round-to-even + if f - i == 0.0 and i % 2: + i -= 1 + else: + f -= 0.5 + i = int(f) + if f - i == 0.0 and i % 2: + i += 1 + return int2bitstore(i, 8, True) diff --git a/bitstring/bitstore_tibs.py b/bitstring/bitstore_tibs.py index f11c42aa..b2efa5a2 100644 --- a/bitstring/bitstore_tibs.py +++ b/bitstring/bitstore_tibs.py @@ -3,452 +3,494 @@ from tibs import Tibs, Mutibs from bitstring.exceptions import CreationError -from typing import Union, Iterable, Optional, overload, Iterator, Any -from bitstring.helpers import offset_slice_indices_lsb0 +from typing import Any +from collections.abc import Iterable, Iterator +def _fallback_to_u(tibs: Tibs | Mutibs) -> int: + if len(tibs) == 0: + # Keep tibs' validation behaviour for zero-length conversion. + return tibs.to_u() + padding = (-len(tibs)) % 8 + if padding: + tibs = [0] * padding + tibs + return int.from_bytes(tibs.to_bytes(), byteorder="big", signed=False) + + +def _fallback_to_i(tibs: Tibs | Mutibs) -> int: + if len(tibs) == 0: + # Keep tibs' validation behaviour for zero-length conversion. + return tibs.to_i() + padding = (-len(tibs)) % 8 + if padding: + pad_bit = tibs[0] + tibs = [pad_bit] * padding + tibs + return int.from_bytes(tibs.to_bytes(), byteorder="big", signed=True) + + +def _to_u(tibs: Tibs | Mutibs) -> int: + if len(tibs) <= 128: + return tibs.to_u() + return _fallback_to_u(tibs) + + +def _to_i(tibs: Tibs | Mutibs) -> int: + if len(tibs) <= 128: + return tibs.to_i() + return _fallback_to_i(tibs) + class ConstBitStore: - """A light wrapper around tibs.Tibs that does the LSB0 stuff""" + """A light wrapper around tibs.Tibs""" - __slots__ = ('_bits',) + __slots__ = ('tibs',) - def __init__(self, initializer: Union[Tibs, None] = None) -> None: - if initializer is not None: - self._bits = initializer - else: - self._bits = Tibs() + def __init__(self, initializer: Tibs) -> None: + self.tibs = initializer @classmethod def join(cls, bitstores: Iterable[ConstBitStore], /) -> ConstBitStore: x = super().__new__(cls) - x._bits = Tibs.from_joined(b._bits for b in bitstores) + x.tibs = Tibs.from_joined(b.tibs for b in bitstores) return x @classmethod def from_zeros(cls, i: int): x = super().__new__(cls) - x._bits = Tibs.from_zeros(i) + x.tibs = Tibs.from_zeros(i) return x @classmethod - def from_tibs(cls, tb: Tibs): + def from_ones(cls, i: int): x = super().__new__(cls) - x._bits = tb + x.tibs = Tibs.from_ones(i) return x @classmethod - def from_bytes(cls, b: Union[bytes, bytearray, memoryview], /) -> ConstBitStore: + def from_bytes(cls, b: bytes | bytearray | memoryview, /, offset: int | None = None, + length: int | None = None) -> ConstBitStore: x = super().__new__(cls) - x._bits = Tibs.from_bytes(b) + x.tibs = Tibs.from_bytes(b, offset=offset, length=length) return x @classmethod - def frombuffer(cls, buffer, /, length: Optional[int] = None) -> ConstBitStore: + def from_bools(cls, iterable: Iterable[Any], /) -> ConstBitStore: x = super().__new__(cls) - # TODO: tibs needs a Tibs.from_buffer method. - x._bits = Tibs.from_bytes(bytes(buffer)) + x.tibs = Tibs.from_bools(iterable) + return x + + @classmethod + def frombuffer(cls, buffer, /, length: int | None = None, offset: int | None = None) -> ConstBitStore: #TODO: Shouldn't need a default here. + mv = memoryview(buffer) + if offset is None: + offset = 0 + if offset < 0: + raise CreationError("Can't create bitstring with a negative offset.") + if offset > mv.nbytes * 8: + raise CreationError( + f"Can't create bitstring with an offset of {offset} from {mv.nbytes * 8} bits of data.") if length is not None: if length < 0: raise CreationError("Can't create bitstring with a negative length.") - if length > len(x._bits): + if offset + length > mv.nbytes * 8: raise CreationError( - f"Can't create bitstring with a length of {length} from {len(x._bits)} bits of data.") - return x.getslice(0, length) if length is not None else x + f"Can't create bitstring with a length of {length} from {mv.nbytes * 8 - offset} bits of data.") + return cls.from_bytes(mv, offset=offset, length=length) + return cls.from_bytes(mv, offset=offset) @classmethod def from_bin(cls, s: str) -> ConstBitStore: x = super().__new__(cls) - x._bits = Tibs.from_bin(s) + x.tibs = Tibs.from_bin(s) return x - def set(self, value, pos) -> None: - self._bits.set(value, pos) - - @staticmethod - def using_rust_core() -> bool: - return True + def to_bytes(self) -> bytes: + return self.tibs.to_padded_bytes() - def tobitarray(self): - raise TypeError("tobitarray() is not available when using the Rust core option.") + def read_bytes(self, start: int, length: int) -> bytes: + return self.tibs.to_bytes(start, start + length) - def to_bytes(self, pad_at_end: bool = True) -> bytes: - excess_bits = len(self._bits) % 8 - if excess_bits != 0: - # Pad with zeros to make full bytes - if pad_at_end: - padded_bits = self._bits.to_mutibs().extend(Mutibs.from_zeros(8 - excess_bits)) - else: - padded_bits = self._bits.to_mutibs().extend_left(Mutibs.from_zeros(8 - excess_bits)) - return padded_bits.to_bytes() - return self._bits.to_bytes() + def byte_swapped(self, start: int | None = None, end: int | None = None) -> ConstBitStore: + return ConstBitStore(self.tibs.byte_swapped(start=start, end=end)) def to_u(self) -> int: - if len(self) > 128: - return int.from_bytes(self.to_bytes(pad_at_end=False), byteorder="big", signed=False) - try: - return self._bits.to_u() - except OverflowError as e: - raise ValueError(e) + return _to_u(self.tibs) + + def read_u(self, start: int, length: int) -> int: + if length <= 128: + return self.tibs.to_u(start, start + length) + return _fallback_to_u(self.tibs[start:start + length]) def to_i(self) -> int: - if len(self) > 128: - return int.from_bytes(self.to_bytes(pad_at_end=False), byteorder="big", signed=True) - try: - return self._bits.to_i() - except OverflowError as e: - raise ValueError(e) + return _to_i(self.tibs) + + def read_i(self, start: int, length: int) -> int: + if length <= 128: + return self.tibs.to_i(start, start + length) + return _fallback_to_i(self.tibs[start:start + length]) def to_hex(self) -> str: - return self._bits.to_hex() + return self.tibs.to_hex() + + def read_hex(self, start: int, length: int) -> str: + return self.tibs.to_hex(start, start + length) def to_bin(self) -> str: - return self._bits.to_bin() + return self.tibs.to_bin() + + def read_bin(self, start: int, length: int) -> str: + return self.tibs.to_bin(start, start + length) def to_oct(self) -> str: - return self._bits.to_oct() + return self.tibs.to_oct() + + def read_oct(self, start: int, length: int) -> str: + return self.tibs.to_oct(start, start + length) def __add__(self, other: ConstBitStore, /) -> ConstBitStore: - newbits = self._bits + other._bits - return ConstBitStore.from_tibs(newbits) + return ConstBitStore(self.tibs + other.tibs) def __eq__(self, other: Any, /) -> bool: - return self._bits == other._bits + if not isinstance(other, ConstBitStore): + return NotImplemented + return self.tibs == other.tibs def __and__(self, other: ConstBitStore, /) -> ConstBitStore: - return ConstBitStore.from_tibs(self._bits & other._bits) + return ConstBitStore(self.tibs & other.tibs) def __or__(self, other: ConstBitStore, /) -> ConstBitStore: - return ConstBitStore.from_tibs(self._bits | other._bits) + return ConstBitStore(self.tibs | other.tibs) def __xor__(self, other: ConstBitStore, /) -> ConstBitStore: - return ConstBitStore.from_tibs(self._bits ^ other._bits) + return ConstBitStore(self.tibs ^ other.tibs) def __invert__(self) -> ConstBitStore: - return ConstBitStore.from_tibs(~self._bits) + return ConstBitStore(~self.tibs) + + def __lshift__(self, n: int, /) -> ConstBitStore: + return ConstBitStore(self.tibs << n) + + def __rshift__(self, n: int, /) -> ConstBitStore: + return ConstBitStore(self.tibs >> n) def find(self, bs: ConstBitStore, start: int, end: int, bytealigned: bool = False) -> int | None: - assert start >= 0 - return self._bits.find(bs._bits, start, end, byte_aligned=bytealigned) + return self.tibs.find(bs.tibs, start, end, byte_aligned=bytealigned) def rfind(self, bs: ConstBitStore, start: int, end: int, bytealigned: bool = False) -> int | None: - assert start >= 0 - return self._bits.rfind(bs._bits, start, end, byte_aligned=bytealigned) + return self.tibs.rfind(bs.tibs, start, end, byte_aligned=bytealigned) - def findall_msb0(self, bs: ConstBitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: - x = self._bits - for p in x.find_all(bs._bits, start=start, end=end, byte_aligned=bytealigned): - yield p + def findall(self, bs: ConstBitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: + return self.tibs.find_all_iter(bs.tibs, start=start, end=end, byte_aligned=bytealigned) - def rfindall_msb0(self, bs: ConstBitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: - x = self._bits - for p in x.rfind_all(bs._bits, start=start, end=end, byte_aligned=bytealigned): - yield p - - def count(self, value, /) -> int: - return self._bits.count(value) + def rfindall(self, bs: ConstBitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: + return self.tibs.rfind_all_iter(bs.tibs, start=start, end=end, byte_aligned=bytealigned) def __iter__(self) -> Iterable[bool]: - length = len(self) - for i in range(length): - yield self.getindex(i) + return self.tibs.__iter__() def _mutable_copy(self) -> MutableBitStore: """Always creates a copy, even if instance is immutable.""" - return MutableBitStore.from_mutibs(self._bits.to_mutibs()) + return MutableBitStore(self.tibs.to_mutibs()) def copy(self) -> ConstBitStore: - return self if isinstance(self._bits, Tibs) else self._mutable_copy() + return self - def __getitem__(self, item: Union[int, slice], /) -> Union[int, ConstBitStore]: + def __getitem__(self, item: int | slice, /) -> int | ConstBitStore: # Use getindex or getslice instead raise NotImplementedError - def getindex_msb0(self, index: int, /) -> bool: - return self._bits.__getitem__(index) + def getindex(self, index: int, /) -> bool: + return self.tibs[index] - def getslice_withstep_msb0(self, key: slice, /) -> ConstBitStore: - return ConstBitStore(self._bits.__getitem__(key)) + def getslice_withstep(self, key: slice, /) -> ConstBitStore: + return ConstBitStore(self.tibs[key]) - def getslice_withstep_lsb0(self, key: slice, /) -> ConstBitStore: - key = offset_slice_indices_lsb0(key, len(self)) - return ConstBitStore(self._bits.__getitem__(key)) + def getslice(self, start: int | None, stop: int | None, /) -> ConstBitStore: + return ConstBitStore(self.tibs[start:stop]) - def getslice_msb0(self, start: Optional[int], stop: Optional[int], /) -> ConstBitStore: - return ConstBitStore(self._bits[start:stop]) + def any(self) -> bool: + return self.tibs.any() - def getslice_lsb0(self, start: Optional[int], stop: Optional[int], /) -> ConstBitStore: - s = offset_slice_indices_lsb0(slice(start, stop, None), len(self)) - return ConstBitStore(self._bits[s.start:s.stop]) + def all(self) -> bool: + return self.tibs.all() - def getindex_lsb0(self, index: int, /) -> bool: - return self._bits.__getitem__(-index - 1) + def startswith(self, prefix: ConstBitStore) -> bool: + return self.tibs.starts_with(prefix.tibs) - def any(self) -> bool: - return self._bits.any() + def endswith(self, suffix: ConstBitStore) -> bool: + return self.tibs.ends_with(suffix.tibs) - def all(self) -> bool: - return self._bits.all() + def count(self, value: Any) -> int: + return self.tibs.count(value) def __len__(self) -> int: - return len(self._bits) + return len(self.tibs) class MutableBitStore: - """A light wrapper around tibs.Mutibs that does the LSB0 stuff""" + """A light wrapper around tibs.Mutibs""" - __slots__ = ('_bits',) + __slots__ = ('tibs',) - def __init__(self, initializer: Union[Mutibs, Tibs, None] = None) -> None: - if initializer is not None: - self._bits = initializer - else: - self._bits = Mutibs() + def __init__(self, initializer: Mutibs) -> None: + self.tibs = initializer @classmethod - def from_zeros(cls, i: int): + def join(cls, bitstores: Iterable[MutableBitStore], /) -> MutableBitStore: x = super().__new__(cls) - x._bits = Mutibs.from_zeros(i) + x.tibs = Mutibs.from_joined(b.tibs for b in bitstores) return x @classmethod - def from_mutibs(cls, mb: Mutibs): - assert isinstance(mb, Mutibs) + def from_zeros(cls, i: int): x = super().__new__(cls) - x._bits = mb + x.tibs = Mutibs.from_zeros(i) return x @classmethod - def from_bytes(cls, b: Union[bytes, bytearray, memoryview], /) -> MutableBitStore: + def from_ones(cls, i: int): x = super().__new__(cls) - x._bits = Mutibs.from_bytes(b) + x.tibs = Mutibs.from_ones(i) return x @classmethod - def frombuffer(cls, buffer, /, length: Optional[int] = None) -> MutableBitStore: + def from_bytes(cls, b: bytes | bytearray | memoryview, /, offset: int | None = None, + length: int | None = None) -> MutableBitStore: x = super().__new__(cls) - # TODO: tibs needs a Bits.from_buffer method. - x._bits = Mutibs.from_bytes(bytes(buffer)) + x.tibs = Mutibs.from_bytes(b, offset=offset, length=length) + return x + + @classmethod + def frombuffer(cls, buffer, /, length: int | None = None, offset: int | None = None) -> MutableBitStore: + mv = memoryview(buffer) + if offset is None: + offset = 0 + if offset < 0: + raise CreationError("Can't create bitstring with a negative offset.") + if offset > mv.nbytes * 8: + raise CreationError( + f"Can't create bitstring with an offset of {offset} from {mv.nbytes * 8} bits of data.") if length is not None: if length < 0: raise CreationError("Can't create bitstring with a negative length.") - if length > len(x._bits): + if offset + length > mv.nbytes * 8: raise CreationError( - f"Can't create bitstring with a length of {length} from {len(x._bits)} bits of data.") - return x.getslice(0, length) if length is not None else x + f"Can't create bitstring with a length of {length} bits from {mv.nbytes * 8 - offset} bits of data.") + return cls.from_bytes(mv, offset=offset, length=length) + return cls.from_bytes(mv, offset=offset) @classmethod - def from_bin(cls, s: str) -> MutableBitStore: + def from_bools(cls, iterable: Iterable[Any], /) -> MutableBitStore: x = super().__new__(cls) - x._bits = Mutibs.from_bin(s) + x.tibs = Mutibs.from_bools(iterable) return x - def set(self, value, pos) -> None: - self._bits.set(value, pos) - - @staticmethod - def using_rust_core() -> bool: - return True - - def tobitarray(self): - raise TypeError("tobitarray() is not available when using the Rust core option.") + @classmethod + def from_bin(cls, s: str) -> MutableBitStore: + x = super().__new__(cls) + x.tibs = Mutibs.from_bin(s) + return x def to_bytes(self, pad_at_end: bool = True) -> bytes: - excess_bits = len(self._bits) % 8 + if pad_at_end: + return self.tibs.to_padded_bytes() + excess_bits = len(self.tibs) % 8 if excess_bits != 0: - # Pad with zeros to make full bytes - if pad_at_end: - padded_bits = self._bits + Mutibs.from_zeros(8 - excess_bits) - else: - padded_bits = Mutibs.from_zeros(8 - excess_bits) + self._bits + padded_bits = Mutibs.from_zeros(8 - excess_bits) + self.tibs return padded_bits.to_bytes() - return self._bits.to_bytes() + return self.tibs.to_bytes() + + def read_bytes(self, start: int, length: int) -> bytes: + return self.tibs.to_bytes(start, start + length) + + def byte_swapped(self, start: int | None = None, end: int | None = None) -> MutableBitStore: + return MutableBitStore(self.tibs.byte_swapped(start=start, end=end)) def to_u(self) -> int: - if len(self) > 128: - return int.from_bytes(self.to_bytes(pad_at_end=False), byteorder="big", signed=False) - try: - return self._bits.to_u() - except OverflowError as e: - raise ValueError(e) + return _to_u(self.tibs) + + def read_u(self, start: int, length: int) -> int: + if length <= 128: + return self.tibs.to_u(start, start + length) + return _fallback_to_u(self.tibs[start:start + length]) def to_i(self) -> int: - if len(self) > 128: - return int.from_bytes(self.to_bytes(pad_at_end=False), byteorder="big", signed=True) - try: - return self._bits.to_i() - except OverflowError as e: - raise ValueError(e) + return _to_i(self.tibs) + + def read_i(self, start: int, length: int) -> int: + if length <= 128: + return self.tibs.to_i(start, start + length) + return _fallback_to_i(self.tibs[start:start + length]) def to_hex(self) -> str: - return self._bits.to_hex() + return self.tibs.to_hex() + + def read_hex(self, start: int, length: int) -> str: + return self.tibs.to_hex(start, start + length) def to_bin(self) -> str: - return self._bits.to_bin() + return self.tibs.to_bin() - def to_oct(self) -> str: - return self._bits.to_oct() + def read_bin(self, start: int, length: int) -> str: + return self.tibs.to_bin(start, start + length) - def __imul__(self, n: int, /) -> None: - self._bits *= n + def to_oct(self) -> str: + return self.tibs.to_oct() - def __ilshift__(self, n: int, /) -> None: - self._bits <<= n + def read_oct(self, start: int, length: int) -> str: + return self.tibs.to_oct(start, start + length) - def __irshift__(self, n: int, /) -> None: - self._bits >>= n + def __ilshift__(self, n: int, /) -> MutableBitStore: + self.tibs <<= n + return self - def __iadd__(self, other: MutableBitStore, /) -> MutableBitStore: - self._bits += other._bits + def __irshift__(self, n: int, /) -> MutableBitStore: + self.tibs >>= n return self def __add__(self, other: MutableBitStore, /) -> MutableBitStore: - bs = self._mutable_copy() - bs += other - return bs + return MutableBitStore(self.tibs + other.tibs) + + def __iadd__(self, other: MutableBitStore | ConstBitStore, /) -> MutableBitStore: + self.tibs += other.tibs + return self def __eq__(self, other: Any, /) -> bool: - return self._bits == other._bits + if not isinstance(other, (MutableBitStore, ConstBitStore)): + return NotImplemented + return self.tibs == other.tibs def __and__(self, other: MutableBitStore, /) -> MutableBitStore: - return MutableBitStore.from_mutibs(self._bits & other._bits) + return MutableBitStore(self.tibs & other.tibs) - def __or__(self, other: MutableBitStore, /) -> MutableBitStore: - return MutableBitStore.from_mutibs(self._bits | other._bits) + def __iand__(self, other: MutableBitStore | ConstBitStore, /) -> MutableBitStore: + self.tibs &= other.tibs + return self - def __xor__(self, other: MutableBitStore, /) -> MutableBitStore: - return MutableBitStore.from_mutibs(self._bits ^ other._bits) + def __or__(self, other: MutableBitStore, /) -> MutableBitStore: + return MutableBitStore(self.tibs | other.tibs) - def __iand__(self, other: MutableBitStore, /) -> MutableBitStore: - self._bits &= other._bits + def __ior__(self, other: MutableBitStore | ConstBitStore, /) -> MutableBitStore: + self.tibs |= other.tibs return self - def __ior__(self, other: MutableBitStore, /) -> MutableBitStore: - self._bits |= other._bits - return self + def __xor__(self, other: MutableBitStore, /) -> MutableBitStore: + return MutableBitStore(self.tibs ^ other.tibs) - def __ixor__(self, other: MutableBitStore, /) -> MutableBitStore: - self._bits ^= other._bits + def __ixor__(self, other: MutableBitStore | ConstBitStore, /) -> MutableBitStore: + self.tibs ^= other.tibs return self def __invert__(self) -> MutableBitStore: - return MutableBitStore.from_mutibs(~self._bits) + return MutableBitStore(~self.tibs) + + def __lshift__(self, n: int, /) -> MutableBitStore: + return MutableBitStore(self.tibs << n) - def find(self, bs: MutableBitStore, start: int, end: int, bytealigned: bool = False) -> int: - assert start >= 0 - return self._bits.find(bs._bits, start, end, byte_aligned=bytealigned) + def __rshift__(self, n: int, /) -> MutableBitStore: + return MutableBitStore(self.tibs >> n) - def rfind(self, bs: MutableBitStore, start: int, end: int, bytealigned: bool = False): - assert start >= 0 - return self._bits.rfind(bs._bits, start, end, byte_aligned=bytealigned) + def find(self, bs: MutableBitStore, start: int, end: int, bytealigned: bool = False) -> int | None: + return self.tibs.find(bs.tibs, start, end, byte_aligned=bytealigned) - def findall_msb0(self, bs: MutableBitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: - x = self._bits.to_tibs() - for p in x.find_all(bs._bits, start=start, end=end, byte_aligned=bytealigned): - yield p + def rfind(self, bs: MutableBitStore, start: int, end: int, bytealigned: bool = False) -> int | None: + return self.tibs.rfind(bs.tibs, start, end, byte_aligned=bytealigned) - def rfindall_msb0(self, bs: MutableBitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: - x = self._bits.to_tibs() - for p in x.rfind_all(bs._bits, start=start, end=end, byte_aligned=bytealigned): - yield p + def findall(self, bs: MutableBitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: + return self.tibs.to_tibs().find_all_iter(bs.tibs, start=start, end=end, byte_aligned=bytealigned) - def count(self, value, /) -> int: - return self._bits.count(value) + def rfindall(self, bs: MutableBitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]: + return self.tibs.to_tibs().rfind_all_iter(bs.tibs, start=start, end=end, byte_aligned=bytealigned) def clear(self) -> None: - self._bits.clear() + self.tibs.clear() def reverse(self) -> None: - self._bits.reverse() + self.tibs.reverse() + def byte_swap(self, start: int | None, end: int | None) -> None: + self.tibs.byte_swap(start=start, end=end) + + # TODO: Should we remove iter from this mutable type? def __iter__(self) -> Iterable[bool]: for i in range(len(self)): yield self.getindex(i) - def extend_left(self, other: MutableBitStore, /) -> None: - self._bits.extend_left(other._bits) + def extend_left(self, other: MutableBitStore | ConstBitStore, /) -> None: + self.tibs.extend_left(other.tibs) def _mutable_copy(self) -> MutableBitStore: """Always creates a copy, even if instance is immutable.""" - return MutableBitStore.from_mutibs(self._bits.__copy__()) + return MutableBitStore(self.tibs.__copy__()) def copy(self) -> MutableBitStore: return self._mutable_copy() - def __getitem__(self, item: Union[int, slice], /) -> Union[int, MutableBitStore]: + def __getitem__(self, item: int | slice, /) -> int | MutableBitStore: # Use getindex or getslice instead raise NotImplementedError - def getindex_msb0(self, index: int, /) -> bool: - return self._bits.__getitem__(index) + def getindex(self, index: int, /) -> bool: + return self.tibs[index] - def getslice_withstep_msb0(self, key: slice, /) -> MutableBitStore: - return MutableBitStore(self._bits.__getitem__(key)) + def getslice_withstep(self, key: slice, /) -> MutableBitStore: + return MutableBitStore(self.tibs[key]) - def getslice_withstep_lsb0(self, key: slice, /) -> MutableBitStore: - key = offset_slice_indices_lsb0(key, len(self)) - return MutableBitStore(self._bits.__getitem__(key)) + def getslice(self, start: int | None, stop: int | None, /) -> MutableBitStore: + return MutableBitStore(self.tibs[start:stop]) - def getslice_msb0(self, start: Optional[int], stop: Optional[int], /) -> MutableBitStore: - return MutableBitStore(self._bits[start:stop]) + def invert(self, index: int | None = None, /) -> None: + if index is not None: + self.tibs.invert(index) + else: + self.tibs.invert() - def getslice_lsb0(self, start: Optional[int], stop: Optional[int], /) -> MutableBitStore: - s = offset_slice_indices_lsb0(slice(start, stop, None), len(self)) - return MutableBitStore(self._bits[s.start:s.stop]) + def set(self, value: Any, pos: Any, /) -> None: + if value: + self.tibs.set(pos) + else: + self.tibs.unset(pos) - def getindex_lsb0(self, index: int, /) -> bool: - return self._bits.__getitem__(-index - 1) + def any(self) -> bool: + return self.tibs.any() - @overload - def setitem_lsb0(self, key: int, value: int, /) -> None: - ... + def all(self) -> bool: + return self.tibs.all() - @overload - def setitem_lsb0(self, key: slice, value: MutableBitStore, /) -> None: - ... + def startswith(self, prefix: MutableBitStore | ConstBitStore) -> bool: + return self.tibs.starts_with(prefix.tibs) - def setitem_lsb0(self, key: Union[int, slice], value: Union[int, MutableBitStore], /) -> None: - if isinstance(key, slice): - new_slice = offset_slice_indices_lsb0(key, len(self)) - self._bits.__setitem__(new_slice, value._bits) - else: - self._bits.__setitem__(-key - 1, bool(value)) + def endswith(self, suffix: MutableBitStore | ConstBitStore) -> bool: + return self.tibs.ends_with(suffix.tibs) - def delitem_lsb0(self, key: Union[int, slice], /) -> None: - if isinstance(key, slice): - new_slice = offset_slice_indices_lsb0(key, len(self)) - self._bits.__delitem__(new_slice) - else: - self._bits.__delitem__(-key - 1) + def count(self, value: Any) -> int: + return self.tibs.count(value) - def invert_msb0(self, index: Optional[int] = None, /) -> None: - if index is not None: - self._bits.invert(index) - else: - self._bits.invert() + def replace(self, old: MutableBitStore | ConstBitStore, new: MutableBitStore | ConstBitStore, + start: int | None = None, end: int | None = None, + count: int | None = None, bytealigned: bool = False) -> int: + return self.tibs.replace(old.tibs, new.tibs, start=start, end=end, count=count, + byte_aligned=bytealigned) - def invert_lsb0(self, index: Optional[int] = None, /) -> None: - if index is not None: - self._bits.invert(-index - 1) - else: - self._bits.invert() + def rotate_left(self, n: int, start: int | None = None, end: int | None = None) -> None: + self.tibs.rotate_left(n, start=start, end=end) - def any(self) -> bool: - return self._bits.any() - - def all(self) -> bool: - return self._bits.all() + def rotate_right(self, n: int, start: int | None = None, end: int | None = None) -> None: + self.tibs.rotate_right(n, start=start, end=end) def __len__(self) -> int: - return len(self._bits) + return len(self.tibs) - def setitem_msb0(self, key, value, /): + def __setitem__(self, key, value, /): if isinstance(value, (MutableBitStore, ConstBitStore)): - self._bits.__setitem__(key, value._bits) + self.tibs.__setitem__(key, value.tibs) else: if isinstance(key, slice): key = range(*key.indices(len(self))) - self._bits.set(value, key) + if value: + self.tibs.set(key) + else: + self.tibs.unset(key) - def delitem_msb0(self, key, /): - self._bits.__delitem__(key) + def __delitem__(self, key, /): + self.tibs.__delitem__(key) diff --git a/bitstring/bitstore_tibs_helpers.py b/bitstring/bitstore_tibs_helpers.py deleted file mode 100644 index 3b4f7f54..00000000 --- a/bitstring/bitstore_tibs_helpers.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -from typing import Union -from tibs import Tibs, Mutibs -import bitstring - -MutableBitStore = bitstring.bitstore.MutableBitStore -ConstBitStore = bitstring.bitstore.ConstBitStore - -from bitstring.helpers import tidy_input_string - - -def bin2bitstore(binstring: str) -> ConstBitStore: - binstring = tidy_input_string(binstring) - binstring = binstring.replace('0b', '') - mb = Tibs.from_bin(binstring) - return ConstBitStore.from_tibs(mb) - - -def hex2bitstore(hexstring: str) -> ConstBitStore: - hexstring = tidy_input_string(hexstring) - hexstring = hexstring.replace('0x', '') - mb = Tibs.from_hex(hexstring) - return ConstBitStore.from_tibs(mb) - - -def oct2bitstore(octstring: str) -> ConstBitStore: - octstring = tidy_input_string(octstring) - octstring = octstring.replace('0o', '') - mb = Tibs.from_oct(octstring) - return ConstBitStore.from_tibs(mb) - - -def int2bitstore(i: int, length: int, signed: bool) -> ConstBitStore: - i = int(i) - if length <= 128: - try: - if signed: - mb = Tibs.from_i(i, length=length) - else: - mb = Tibs.from_u(i, length=length) - except OverflowError as e: - raise ValueError(e) - else: - b = i.to_bytes((length + 7) // 8, byteorder="big", signed=signed) - offset = 8 - (length % 8) - mb = Tibs.from_bytes(b) - if offset != 8: - mb = mb[offset:] - return ConstBitStore.from_tibs(mb) - - -def intle2bitstore(i: int, length: int, signed: bool) -> ConstBitStore: - i = int(i) - if length <= 128: - try: - if signed: - mb = Mutibs.from_i(i, length=length) - else: - mb = Mutibs.from_u(i, length=length) - except OverflowError as e: - raise ValueError(e) - mb.byte_swap() - else: - b = i.to_bytes((length + 7) // 8, byteorder="little", signed=signed) - offset = 8 - (length % 8) - mb = Mutibs.from_bytes(b) - if offset != 8: - mb = mb[offset:] - return ConstBitStore.from_tibs(mb.as_tibs()) - - -def float2bitstore(f: Union[str, float], length: int, big_endian: bool) -> ConstBitStore: - f = float(f) - mb = Mutibs.from_f(f, length) - if not big_endian: - mb.byte_swap() - return ConstBitStore.from_tibs(mb.as_tibs()) diff --git a/bitstring/bitstream.py b/bitstring/bitstream.py deleted file mode 100644 index 3d10bda1..00000000 --- a/bitstring/bitstream.py +++ /dev/null @@ -1,726 +0,0 @@ -from __future__ import annotations - -import bitstring -from bitstring.bits import Bits, BitsType -from bitstring.dtypes import Dtype -from typing import Union, List, Any, Optional, overload, TypeVar, Tuple -import copy -import numbers - -common_helpers = bitstring.bitstore_common_helpers - -TConstBitStream = TypeVar("TConstBitStream", bound='ConstBitStream') - - -class ConstBitStream(Bits): - """A container or stream holding an immutable sequence of bits. - - For a mutable container use the BitStream class instead. - - Methods inherited from Bits: - - all() -- Check if all specified bits are set to 1 or 0. - any() -- Check if any of specified bits are set to 1 or 0. - copy() -- Return a copy of the bitstring. - count() -- Count the number of bits set to 1 or 0. - cut() -- Create generator of constant sized chunks. - endswith() -- Return whether the bitstring ends with a sub-string. - find() -- Find a sub-bitstring in the current bitstring. - findall() -- Find all occurrences of a sub-bitstring in the current bitstring. - fromstring() -- Create a bitstring from a formatted string. - join() -- Join bitstrings together using current bitstring. - pp() -- Pretty print the bitstring. - rfind() -- Seek backwards to find a sub-bitstring. - split() -- Create generator of chunks split by a delimiter. - startswith() -- Return whether the bitstring starts with a sub-bitstring. - tobitarray() -- Return bitstring as a bitarray from the bitarray package. - tobytes() -- Return bitstring as bytes, padding if needed. - tofile() -- Write bitstring to file, padding if needed. - unpack() -- Interpret bits using format string. - - Other methods: - - bytealign() -- Align to next byte boundary. - peek() -- Peek at and interpret next bits as a single item. - peeklist() -- Peek at and interpret next bits as a list of items. - read() -- Read and interpret next bits as a single item. - readlist() -- Read and interpret next bits as a list of items. - readto() -- Read up to and including next occurrence of a bitstring. - - Special methods: - - Also available are the operators [], ==, !=, +, *, ~, <<, >>, &, |, ^. - - Properties: - - [GENERATED_PROPERTY_DESCRIPTIONS] - - len -- Length of the bitstring in bits. - pos -- The current bit position in the bitstring. - """ - - __slots__ = ('_pos',) - - def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None, - offset: Optional[int] = None, pos: int = 0, **kwargs) -> None: - """Either specify an 'auto' initialiser: - A string of comma separated tokens, an integer, a file object, - a bytearray, a boolean iterable or another bitstring. - - Or initialise via **kwargs with one (and only one) of: - bin -- binary string representation, e.g. '0b001010'. - hex -- hexadecimal string representation, e.g. '0x2ef' - oct -- octal string representation, e.g. '0o777'. - bytes -- raw data as a bytes object, for example read from a binary file. - int -- a signed integer. - uint -- an unsigned integer. - float / floatbe -- a big-endian floating point number. - bool -- a boolean (True or False). - se -- a signed exponential-Golomb code. - ue -- an unsigned exponential-Golomb code. - sie -- a signed interleaved exponential-Golomb code. - uie -- an unsigned interleaved exponential-Golomb code. - floatle -- a little-endian floating point number. - floatne -- a native-endian floating point number. - bfloat / bfloatbe - a big-endian bfloat format 16-bit floating point number. - bfloatle -- a little-endian bfloat format 16-bit floating point number. - bfloatne -- a native-endian bfloat format 16-bit floating point number. - intbe -- a signed big-endian whole byte integer. - intle -- a signed little-endian whole byte integer. - intne -- a signed native-endian whole byte integer. - uintbe -- an unsigned big-endian whole byte integer. - uintle -- an unsigned little-endian whole byte integer. - uintne -- an unsigned native-endian whole byte integer. - filename -- the path of a file which will be opened in binary read-only mode. - - Other keyword arguments: - length -- length of the bitstring in bits, if needed and appropriate. - It must be supplied for all integer and float initialisers. - offset -- bit offset to the data. These offset bits are - ignored and this is mainly intended for use when - initialising using 'bytes' or 'filename'. - pos -- Initial bit position, defaults to 0. - - """ - pass - - def __new__(cls, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None, - offset: Optional[int] = None, pos: int = 0, **kwargs) -> TConstBitStream: - x = super().__new__(cls, auto, length, offset, **kwargs) - if pos < 0: - pos += len(x._bitstore) - if pos < 0 or pos > len(x._bitstore): - raise bitstring.CreationError(f"Cannot set pos to {pos} when length is {len(x._bitstore)}.") - x._pos = pos - return x - - def _setbytepos(self, bytepos: int) -> None: - """Move to absolute byte-aligned position in stream.""" - self._setbitpos(bytepos * 8) - - def _getbytepos(self) -> int: - """Return the current position in the stream in bytes. Must be byte aligned.""" - if self._pos % 8: - raise bitstring.ByteAlignError("Not byte aligned when using bytepos property.") - return self._pos // 8 - - def _setbitpos(self, pos: int) -> None: - """Move to absolute position bit in bitstream.""" - if pos < 0: - raise ValueError("Bit position cannot be negative.") - if pos > len(self): - raise ValueError("Cannot seek past the end of the data.") - self._pos = pos - - def _getbitpos(self) -> int: - """Return the current position in the stream in bits.""" - return self._pos - - def _clear(self) -> None: - Bits._clear(self) - self._pos = 0 - - def __copy__(self: TConstBitStream) -> TConstBitStream: - """Return a new copy of the ConstBitStream for the copy module.""" - # Note that if you want a new copy (different ID), use _copy instead. - # The copy can use the same datastore as it's immutable. - s = self.__class__() - s._bitstore = self._bitstore - # Reset the bit position, don't copy it. - s._pos = 0 - return s - - def __and__(self: TConstBitStream, bs: BitsType, /) -> TConstBitStream: - """Bit-wise 'and' between two bitstrings. Returns new bitstring. - - bs -- The bitstring to '&' with. - - Raises ValueError if the two bitstrings have differing lengths. - - """ - s = Bits.__and__(self, bs) - s._pos = 0 - return s - - def __or__(self: TConstBitStream, bs: BitsType, /) -> TConstBitStream: - """Bit-wise 'or' between two bitstrings. Returns new bitstring. - - bs -- The bitstring to '|' with. - - Raises ValueError if the two bitstrings have differing lengths. - - """ - s = Bits.__or__(self, bs) - s._pos = 0 - return s - - def __xor__(self: TConstBitStream, bs: BitsType, /) -> TConstBitStream: - """Bit-wise 'xor' between two bitstrings. Returns new bitstring. - - bs -- The bitstring to '^' with. - - Raises ValueError if the two bitstrings have differing lengths. - - """ - s = Bits.__xor__(self, bs) - s._pos = 0 - return s - - def __add__(self: TConstBitStream, bs: BitsType, /) -> TConstBitStream: - """Concatenate bitstrings and return new bitstring. - - bs -- the bitstring to append. - - """ - s = Bits.__add__(self, bs) - s._pos = 0 - return s - - def append(self, bs: BitsType, /) -> None: - """Append a bitstring to the current bitstring. - - bs -- The bitstring to append. - - The current bit position will be moved to the end of the BitStream. - - """ - self._append(bs) - self._pos = len(self) - - def __repr__(self) -> str: - """Return representation that could be used to recreate the bitstring. - - If the returned string is too long it will be truncated. See __str__(). - - """ - return self._repr(self.__class__.__name__, len(self), self._pos) - - def overwrite(self, bs: BitsType, /, pos: Optional[int] = None) -> None: - """Overwrite with bitstring at bit position pos. - - bs -- The bitstring to overwrite with. - pos -- The bit position to begin overwriting from. - - The current bit position will be moved to the end of the overwritten section. - Raises ValueError if pos < 0 or pos > len(self). - - """ - bs = Bits._create_from_bitstype(bs) - if len(bs) == 0: - return - if pos is None: - pos = self._pos - if pos < 0: - pos += len(self) - if pos < 0 or pos > len(self): - raise ValueError("Overwrite starts outside boundary of bitstring.") - self._overwrite(bs, pos) - self._pos = pos + len(bs) - - def find(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None, - bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]: - """Find first occurrence of substring bs. - - Returns a single item tuple with the bit position if found, or an - empty tuple if not found. The bit position (pos property) will - also be set to the start of the substring if it is found. - - bs -- The bitstring to find. - start -- The bit position to start the search. Defaults to 0. - end -- The bit position one past the last bit to search. - Defaults to len(self). - bytealigned -- If True the bitstring will only be - found on byte boundaries. - - Raises ValueError if bs is empty, if start < 0, if end > len(self) or - if end < start. - - >>> BitStream('0xc3e').find('0b1111') - (6,) - - """ - - p = super().find(bs, start, end, bytealigned) - if p: - self._pos = p[0] - return p - - def rfind(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None, - bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]: - """Find final occurrence of substring bs. - - Returns a single item tuple with the bit position if found, or an - empty tuple if not found. The bit position (pos property) will - also be set to the start of the substring if it is found. - - bs -- The bitstring to find. - start -- The bit position to end the reverse search. Defaults to 0. - end -- The bit position one past the first bit to reverse search. - Defaults to len(self). - bytealigned -- If True the bitstring will only be found on byte - boundaries. - - Raises ValueError if bs is empty, if start < 0, if end > len(self) or - if end < start. - - """ - p = super().rfind(bs, start, end, bytealigned) - if p: - self._pos = p[0] - return p - - @overload - def read(self, fmt: int) -> Bits: - ... - - @overload - def read(self, fmt: str) -> Any: - ... - - def read(self, fmt: Union[int, str, Dtype]) -> Union[int, float, str, Bits, bool, bytes, None]: - """Interpret next bits according to the format string and return result. - - fmt -- Token string describing how to interpret the next bits. - - Token examples: 'int:12' : 12 bits as a signed integer - 'uint:8' : 8 bits as an unsigned integer - 'float:64' : 8 bytes as a big-endian float - 'intbe:16' : 2 bytes as a big-endian signed integer - 'uintbe:16' : 2 bytes as a big-endian unsigned integer - 'intle:32' : 4 bytes as a little-endian signed integer - 'uintle:32' : 4 bytes as a little-endian unsigned integer - 'floatle:64': 8 bytes as a little-endian float - 'intne:24' : 3 bytes as a native-endian signed integer - 'uintne:24' : 3 bytes as a native-endian unsigned integer - 'floatne:32': 4 bytes as a native-endian float - 'hex:80' : 80 bits as a hex string - 'oct:9' : 9 bits as an octal string - 'bin:1' : single bit binary string - 'ue' : next bits as unsigned exp-Golomb code - 'se' : next bits as signed exp-Golomb code - 'uie' : next bits as unsigned interleaved exp-Golomb code - 'sie' : next bits as signed interleaved exp-Golomb code - 'bits:5' : 5 bits as a bitstring - 'bytes:10' : 10 bytes as a bytes object - 'bool' : 1 bit as a bool - 'pad:3' : 3 bits of padding to ignore - returns None - - fmt may also be an integer, which will be treated like the 'bits' token. - - The position in the bitstring is advanced to after the read items. - - Raises ReadError if not enough bits are available. - Raises ValueError if the format is not understood. - - """ - p = self._pos - if isinstance(fmt, numbers.Integral): - if fmt < 0: - raise ValueError("Cannot read negative amount.") - if fmt > len(self) - self._pos: - raise bitstring.ReadError(f"Cannot read {fmt} bits, only {len(self) - self._pos} available.") - bs = self._slice(self._pos, self._pos + fmt) - self._pos += fmt - return bs - dtype = bitstring.dtypes.Dtype(fmt) - if dtype.bitlength is None and not dtype.variable_length: - # No length specified? Try again, but read to end. - bitlength = len(self) - self._pos - items, remainder = divmod(bitlength, dtype.bits_per_item) - if remainder != 0: - raise ValueError( - f"The '{dtype.name}' type must have a bit length that is a multiple of {dtype.bits_per_item}" - f" so cannot be read from the {bitlength} bits that are available.") - dtype = bitstring.dtypes.Dtype(fmt, items) - if dtype.bitlength is not None: - val = dtype.read_fn(self, self._pos) - self._pos += dtype.bitlength - else: - val, self._pos = dtype.read_fn(self, self._pos) - - if self._pos > len(self): - self._pos = p - raise bitstring.ReadError(f"Reading off end of bitstring with fmt '{fmt}'. Only {len(self) - p} bits available.") - return val - - def readlist(self, fmt: Union[str, List[Union[int, str, Dtype]]], **kwargs) \ - -> List[Union[int, float, str, Bits, bool, bytes, None]]: - """Interpret next bits according to format string(s) and return list. - - fmt -- A single string or list of strings with comma separated tokens - describing how to interpret the next bits in the bitstring. Items - can also be integers, for reading new bitstring of the given length. - kwargs -- A dictionary or keyword-value pairs - the keywords used in the - format string will be replaced with their given value. - - The position in the bitstring is advanced to after the read items. - - Raises ReadError is not enough bits are available. - Raises ValueError if the format is not understood. - - See the docstring for 'read' for token examples. 'pad' tokens are skipped - and not added to the returned list. - - >>> h, b1, b2 = s.readlist('hex:20, bin:5, bin:3') - >>> i, bs1, bs2 = s.readlist(['uint:12', 10, 10]) - - """ - value, self._pos = self._readlist(fmt, self._pos, **kwargs) - return value - - def readto(self: TConstBitStream, bs: BitsType, /, bytealigned: Optional[bool] = None) -> TConstBitStream: - """Read up to and including next occurrence of bs and return result. - - bs -- The bitstring to find. - bytealigned -- If True the bitstring will only be - found on byte boundaries. - - Raises ValueError if bs is empty. - Raises ReadError if bs is not found. - - """ - if isinstance(bs, numbers.Integral): - raise ValueError("Integers cannot be searched for") - bs = Bits._create_from_bitstype(bs) - oldpos = self._pos - p = self.find(bs, self._pos, bytealigned=bytealigned) - if not p: - raise bitstring.ReadError("Substring not found") - self._pos += len(bs) - return self._slice(oldpos, self._pos) - - @overload - def peek(self: TConstBitStream, fmt: int) -> TConstBitStream: - ... - - @overload - def peek(self, fmt: str) -> Union[int, float, str, TConstBitStream, bool, bytes, None]: - ... - - def peek(self: TConstBitStream, fmt: Union[int, str]) -> Union[int, float, str, TConstBitStream, bool, bytes, None]: - """Interpret next bits according to format string and return result. - - fmt -- Token string describing how to interpret the next bits. - - The position in the bitstring is not changed. If not enough bits are - available then all bits to the end of the bitstring will be used. - - Raises ReadError if not enough bits are available. - Raises ValueError if the format is not understood. - - See the docstring for 'read' for token examples. - - """ - pos_before = self._pos - value = self.read(fmt) - self._pos = pos_before - return value - - def peeklist(self, fmt: Union[str, List[Union[int, str]]], **kwargs) \ - -> List[Union[int, float, str, Bits, None]]: - """Interpret next bits according to format string(s) and return list. - - fmt -- One or more integers or strings with comma separated tokens describing - how to interpret the next bits in the bitstring. - kwargs -- A dictionary or keyword-value pairs - the keywords used in the - format string will be replaced with their given value. - - The position in the bitstring is not changed. If not enough bits are - available then all bits to the end of the bitstring will be used. - - Raises ReadError if not enough bits are available. - Raises ValueError if the format is not understood. - - See the docstring for 'read' for token examples. - - """ - pos = self._pos - return_values = self.readlist(fmt, **kwargs) - self._pos = pos - return return_values - - def bytealign(self) -> int: - """Align to next byte and return number of skipped bits. - - Raises ValueError if the end of the bitstring is reached before - aligning to the next byte. - - """ - skipped = (8 - (self._pos % 8)) % 8 - self.pos += skipped - return skipped - - @classmethod - def fromstring(cls: TBits, s: str, /) -> TBits: - x = super().__new__(cls) - x._bitstore = common_helpers.str_to_bitstore(s) - x._pos = 0 - return x - - @overload - def __getitem__(self: TBits, key: slice, /) -> TBits: - ... - - @overload - def __getitem__(self: TBits, key: int, /) -> bool: - ... - - def __getitem__(self: TBits, key: Union[slice, int], /) -> Union[TBits, bool]: - """Return a new bitstring representing a slice of the current bitstring.""" - if isinstance(key, numbers.Integral): - return bool(self._bitstore.getindex(key)) - bs = super().__new__(self.__class__) - bs._bitstore = self._bitstore.getslice_withstep(key) - bs._pos = 0 - return bs - - pos = property(_getbitpos, _setbitpos, - doc="""The position in the bitstring in bits. Read and write. - """) - bitpos = property(_getbitpos, _setbitpos, - doc="""The position in the bitstring in bits. Read and write. - """) - bytepos = property(_getbytepos, _setbytepos, - doc="""The position in the bitstring in bytes. Read and write. - """) - - -class BitStream(ConstBitStream, bitstring.BitArray): - """A container or stream holding a mutable sequence of bits - - Subclass of the ConstBitStream and BitArray classes. Inherits all of - their methods. - - Methods: - - all() -- Check if all specified bits are set to 1 or 0. - any() -- Check if any of specified bits are set to 1 or 0. - append() -- Append a bitstring. - bytealign() -- Align to next byte boundary. - byteswap() -- Change byte endianness in-place. - clear() -- Remove all bits from the bitstring. - copy() -- Return a copy of the bitstring. - count() -- Count the number of bits set to 1 or 0. - cut() -- Create generator of constant sized chunks. - endswith() -- Return whether the bitstring ends with a sub-string. - find() -- Find a sub-bitstring in the current bitstring. - findall() -- Find all occurrences of a sub-bitstring in the current bitstring. - fromstring() -- Create a bitstring from a formatted string. - insert() -- Insert a bitstring. - invert() -- Flip bit(s) between one and zero. - join() -- Join bitstrings together using current bitstring. - overwrite() -- Overwrite a section with a new bitstring. - peek() -- Peek at and interpret next bits as a single item. - peeklist() -- Peek at and interpret next bits as a list of items. - pp() -- Pretty print the bitstring. - prepend() -- Prepend a bitstring. - read() -- Read and interpret next bits as a single item. - readlist() -- Read and interpret next bits as a list of items. - readto() -- Read up to and including next occurrence of a bitstring. - replace() -- Replace occurrences of one bitstring with another. - reverse() -- Reverse bits in-place. - rfind() -- Seek backwards to find a sub-bitstring. - rol() -- Rotate bits to the left. - ror() -- Rotate bits to the right. - set() -- Set bit(s) to 1 or 0. - split() -- Create generator of chunks split by a delimiter. - startswith() -- Return whether the bitstring starts with a sub-bitstring. - tobitarray() -- Return bitstring as a bitarray from the bitarray package. - tobytes() -- Return bitstring as bytes, padding if needed. - tofile() -- Write bitstring to file, padding if needed. - unpack() -- Interpret bits using format string. - - Special methods: - - Mutating operators are available: [], <<=, >>=, +=, *=, &=, |= and ^= - in addition to [], ==, !=, +, *, ~, <<, >>, &, | and ^. - - Properties: - - [GENERATED_PROPERTY_DESCRIPTIONS] - - len -- Length of the bitstring in bits. - pos -- The current bit position in the bitstring. - """ - - __slots__ = () - - def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None, - offset: Optional[int] = None, pos: int = 0, **kwargs) -> None: - """Either specify an 'auto' initialiser: - A string of comma separated tokens, an integer, a file object, - a bytearray, a boolean iterable or another bitstring. - - Or initialise via **kwargs with one (and only one) of: - bin -- binary string representation, e.g. '0b001010'. - hex -- hexadecimal string representation, e.g. '0x2ef' - oct -- octal string representation, e.g. '0o777'. - bytes -- raw data as a bytes object, for example read from a binary file. - int -- a signed integer. - uint -- an unsigned integer. - float / floatbe -- a big-endian floating point number. - bool -- a boolean (True or False). - se -- a signed exponential-Golomb code. - ue -- an unsigned exponential-Golomb code. - sie -- a signed interleaved exponential-Golomb code. - uie -- an unsigned interleaved exponential-Golomb code. - floatle -- a little-endian floating point number. - floatne -- a native-endian floating point number. - bfloat / bfloatbe - a big-endian bfloat format 16-bit floating point number. - bfloatle -- a little-endian bfloat format 16-bit floating point number. - bfloatne -- a native-endian bfloat format 16-bit floating point number. - intbe -- a signed big-endian whole byte integer. - intle -- a signed little-endian whole byte integer. - intne -- a signed native-endian whole byte integer. - uintbe -- an unsigned big-endian whole byte integer. - uintle -- an unsigned little-endian whole byte integer. - uintne -- an unsigned native-endian whole byte integer. - filename -- the path of a file which will be opened in binary read-only mode. - - Other keyword arguments: - length -- length of the bitstring in bits, if needed and appropriate. - It must be supplied for all integer and float initialisers. - offset -- bit offset to the data. These offset bits are - ignored and this is intended for use when - initialising using 'bytes' or 'filename'. - pos -- Initial bit position, defaults to 0. - - """ - ConstBitStream.__init__(self, auto, length, offset, pos, **kwargs) - - @classmethod - def fromstring(cls: TBits, s: str, /) -> TBits: - x = super().__new__(cls) - b = common_helpers.str_to_bitstore(s) - x._bitstore = b._mutable_copy() - x._pos = 0 - return x - - def __copy__(self) -> BitStream: - """Return a new copy of the BitStream.""" - s_copy = object.__new__(BitStream) - s_copy._pos = 0 - s_copy._bitstore = self._bitstore.copy() - return s_copy - - def __iadd__(self, bs: BitsType, /) -> BitStream: - """Append to current bitstring. Return self. - - bs -- the bitstring to append. - - The current bit position will be moved to the end of the BitStream. - """ - self._append(bs) - self._pos = len(self) - return self - - def prepend(self, bs: BitsType, /) -> None: - """Prepend a bitstring to the current bitstring. - - bs -- The bitstring to prepend. - - """ - bs = Bits._create_from_bitstype(bs) - super().prepend(bs) - self._pos = 0 - - def __setitem__(self, /, key: Union[slice, int], value: BitsType) -> None: - length_before = len(self) - super().__setitem__(key, value) - if len(self) != length_before: - self._pos = 0 - return - - def __delitem__(self, /, key: Union[slice, int]) -> None: - """Delete item or range. - - >>> a = BitStream('0x001122') - >>> del a[8:16] - >>> print a - 0x0022 - - """ - length_before = len(self) - self._bitstore.__delitem__(key) - if len(self) != length_before: - self._pos = 0 - - def insert(self, bs: BitsType, /, pos: Optional[int] = None) -> None: - """Insert bitstring at bit position pos. - - bs -- The bitstring to insert. - pos -- The bit position to insert at. - - The current bit position will be moved to the end of the inserted section. - Raises ValueError if pos < 0 or pos > len(self). - - """ - bs = Bits._create_from_bitstype(bs) - if len(bs) == 0: - return - if bs is self: - bs = self._copy() - if pos is None: - pos = self._pos - if pos < 0: - pos += len(self) - if not 0 <= pos <= len(self): - raise ValueError("Invalid insert position.") - self._insert(bs, pos) - self._pos = pos + len(bs) - - def replace(self, old: BitsType, new: BitsType, start: Optional[int] = None, end: Optional[int] = None, - count: Optional[int] = None, bytealigned: Optional[bool] = None) -> int: - """Replace all occurrences of old with new in place. - - Returns number of replacements made. - - old -- The bitstring to replace. - new -- The replacement bitstring. - start -- Any occurrences that start before this will not be replaced. - Defaults to 0. - end -- Any occurrences that finish after this will not be replaced. - Defaults to len(self). - count -- The maximum number of replacements to make. Defaults to - replace all occurrences. - bytealigned -- If True replacements will only be made on byte - boundaries. - - Raises ValueError if old is empty or if start or end are - out of range. - - """ - if count == 0: - return 0 - if len(old := Bits._create_from_bitstype(old)) == 0: - raise ValueError("Empty bitstring cannot be replaced.") - start, end = self._validate_slice(start, end) - new = Bits._create_from_bitstype(new) - if new is self: - # Prevent self assignment woes - new = copy.copy(self) - length_before = len(self) - replacement_count = self._replace(old, new, start, end, 0 if count is None else count, bytealigned) - if len(self) != length_before: - self._pos = 0 - return replacement_count \ No newline at end of file diff --git a/bitstring/bitstring_options.py b/bitstring/bitstring_options.py index 306ea4fb..d7b447c1 100644 --- a/bitstring/bitstring_options.py +++ b/bitstring/bitstring_options.py @@ -1,10 +1,7 @@ from __future__ import annotations -import bitstring import os -ConstBitStore = bitstring.bitstore.ConstBitStore -MutableBitStore = bitstring.bitstore.MutableBitStore class Options: """Internal class to create singleton module options instance.""" @@ -12,74 +9,19 @@ class Options: _instance = None def __init__(self): - self.set_lsb0(False) + if hasattr(self, "_initialised"): + return self._bytealigned = False - self.mxfp_overflow = 'saturate' self.no_color = False no_color = os.getenv('NO_COLOR') self.no_color = True if no_color else False - - @property - def using_rust_core(self): - x = ConstBitStore() - return x.using_rust_core() - - @property - def mxfp_overflow(self) -> str: - return self._mxfp_overflow - - @mxfp_overflow.setter - def mxfp_overflow(self, value: str) -> None: - allowed_values = ('saturate', 'overflow') - if value not in allowed_values: - raise ValueError(f"mxfp_overflow must be one of {allowed_values}, not {value}.") - self._mxfp_overflow = value + self._initialised = True def __repr__(self) -> str: attributes = {attr: getattr(self, attr) for attr in dir(self) if not attr.startswith('_') and not callable(getattr(self, attr))} return '\n'.join(f"{attr}: {value!r}" for attr, value in attributes.items()) - @property - def lsb0(self) -> bool: - return self._lsb0 - - @lsb0.setter - def lsb0(self, value: bool) -> None: - self.set_lsb0(value) - - def set_lsb0(self, value: bool) -> None: - self._lsb0 = bool(value) - Bits = bitstring.bits.Bits - BitArray = bitstring.bitarray_.BitArray - - lsb0_methods = { - Bits: {'_find': Bits._find_lsb0, '_rfind': Bits._rfind_lsb0, '_findall': Bits._findall_lsb0}, - BitArray: {'_ror': BitArray._rol_msb0, '_rol': BitArray._ror_msb0, '_append': BitArray._append_lsb0, - '_prepend': BitArray._append_msb0}, - ConstBitStore: {'__setitem__': None, '__delitem__': None, 'invert': None, - 'getindex': ConstBitStore.getindex_lsb0, 'getslice': ConstBitStore.getslice_lsb0, - 'getslice_withstep': ConstBitStore.getslice_withstep_lsb0}, - MutableBitStore: {'__setitem__': MutableBitStore.setitem_lsb0, '__delitem__': MutableBitStore.delitem_lsb0, - 'getindex': MutableBitStore.getindex_lsb0, 'getslice': MutableBitStore.getslice_lsb0, - 'getslice_withstep': MutableBitStore.getslice_withstep_lsb0, 'invert': MutableBitStore.invert_lsb0} - } - msb0_methods = { - Bits: {'_find': Bits._find_msb0, '_rfind': Bits._rfind_msb0, '_findall': Bits._findall_msb0}, - BitArray: {'_ror': BitArray._ror_msb0, '_rol': BitArray._rol_msb0, '_append': BitArray._append_msb0, - '_prepend': BitArray._append_lsb0}, - ConstBitStore: {'__setitem__': None, '__delitem__': None, 'invert': None, - 'getindex': ConstBitStore.getindex_msb0, 'getslice': ConstBitStore.getslice_msb0, - 'getslice_withstep': ConstBitStore.getslice_withstep_msb0}, - MutableBitStore: {'__setitem__': MutableBitStore.setitem_msb0, '__delitem__': MutableBitStore.delitem_msb0, - 'getindex': MutableBitStore.getindex_msb0, 'getslice': MutableBitStore.getslice_msb0, - 'getslice_withstep': MutableBitStore.getslice_withstep_msb0, 'invert': MutableBitStore.invert_msb0} - } - methods = lsb0_methods if self._lsb0 else msb0_methods - for cls, method_dict in methods.items(): - for attr, method in method_dict.items(): - setattr(cls, attr, method) - @property def bytealigned(self) -> bool: return self._bytealigned @@ -90,7 +32,7 @@ def bytealigned(self, value: bool) -> None: def __new__(cls): if cls._instance is None: - cls._instance = super(Options, cls).__new__(cls) + cls._instance = super().__new__(cls) return cls._instance diff --git a/bitstring/dtypes.py b/bitstring/dtypes.py index 86c4ad1d..e38648c9 100644 --- a/bitstring/dtypes.py +++ b/bitstring/dtypes.py @@ -1,7 +1,8 @@ from __future__ import annotations import functools -from typing import Optional, Dict, Any, Union, Tuple, Callable +from typing import Any +from collections.abc import Callable import inspect import bitstring from bitstring import utils @@ -9,19 +10,19 @@ CACHE_SIZE = 256 -def scaled_get_fn(get_fn, s: Union[int, float]): +def scaled_get_fn(get_fn, s: int | float): def wrapper(*args, scale=s, **kwargs): return get_fn(*args, **kwargs) * scale return wrapper -def scaled_set_fn(set_fn, s: Union[int, float]): +def scaled_set_fn(set_fn, s: int | float): def wrapper(bs, value, *args, scale=s, **kwargs): return set_fn(bs, value / scale, *args, **kwargs) return wrapper -def scaled_read_fn(read_fn, s: Union[int, float]): +def scaled_read_fn(read_fn, s: int | float): def wrapper(*args, scale=s, **kwargs): val = read_fn(*args, **kwargs) if isinstance(val, tuple): @@ -36,8 +37,8 @@ class Dtype: Dtype instances are immutable. They are often created implicitly elsewhere via a token string. - >>> u12 = Dtype('uint', 12) # length separate from token string. - >>> float16 = Dtype('float16') # length part of token string. + >>> u12 = Dtype('u', 12) # length separate from token string. + >>> float16 = Dtype('f16') # length part of token string. >>> mxfp = Dtype('e3m2mxfp', scale=2 ** 6) # dtype with scaling factor """ @@ -50,12 +51,12 @@ class Dtype: _is_signed: bool _set_fn_needs_length: bool _variable_length: bool - _bitlength: Optional[int] + _bitlength: int | None _bits_per_item: int - _length: Optional[int] - _scale: Union[None, float, int] + _length: int | None + _scale: None | float | int - def __new__(cls, token: Union[str, Dtype], /, length: Optional[int] = None, scale: Union[None, float, int] = None) -> Dtype: + def __new__(cls, token: str | Dtype, /, length: int | None = None, scale: None | float | int = None) -> Dtype: if isinstance(token, cls): return token if length is None: @@ -66,7 +67,7 @@ def __new__(cls, token: Union[str, Dtype], /, length: Optional[int] = None, scal return x @property - def scale(self) -> Union[int, float, None]: + def scale(self) -> int | float | None: """The multiplicative scale applied when interpreting the data.""" return self._scale @@ -81,7 +82,7 @@ def length(self) -> int: return self._length @property - def bitlength(self) -> Optional[int]: + def bitlength(self) -> int | None: """The number of bits needed to represent a single instance of the data type. Set to None for variable length dtypes.""" return self._bitlength @@ -97,7 +98,7 @@ def variable_length(self) -> bool: @property def return_type(self) -> Any: - """The type of the value returned by the parse method, such as int, float or str.""" + """The type of the value returned by the unpack method, such as int, float or str.""" return self._return_type @property @@ -106,7 +107,7 @@ def is_signed(self) -> bool: return self._is_signed @property - def set_fn(self) -> Optional[Callable]: + def set_fn(self) -> Callable | None: """A function to set the value of the data type.""" return self._set_fn @@ -120,7 +121,7 @@ def read_fn(self) -> Callable: """A function to read the value of the data type.""" return self._read_fn - def _set_scale(self, value: Union[None, float, int]) -> None: + def _set_scale(self, value: None | float | int) -> None: self._scale = value if self._scale is None: return @@ -136,16 +137,16 @@ def _set_scale(self, value: Union[None, float, int]) -> None: @classmethod @functools.lru_cache(CACHE_SIZE) - def _new_from_token(cls, token: str, scale: Union[None, float, int] = None) -> Dtype: + def _new_from_token(cls, token: str, scale: None | float | int = None) -> Dtype: token = ''.join(token.split()) return dtype_register.get_dtype(*utils.parse_name_length_token(token), scale=scale) def __hash__(self) -> int: - return hash((self._name, self._length)) + return hash((self._name, self._length, self._scale)) @classmethod @functools.lru_cache(CACHE_SIZE) - def _create(cls, definition: DtypeDefinition, length: Optional[int], scale: Union[None, float, int]) -> Dtype: + def _create(cls, definition: DtypeDefinition, length: int | None, scale: None | float | int) -> Dtype: x = super().__new__(cls) x._name = definition.name x._bitlength = x._length = length @@ -171,23 +172,23 @@ def _create(cls, definition: DtypeDefinition, length: Optional[int], scale: Unio x._set_scale(scale) return x - def build(self, value: Any, /) -> bitstring.Bits: - """Create a bitstring from a value. + def pack(self, value: Any, /) -> bitstring.Bits: + """Pack a value into a bitstring. The value parameter should be of a type appropriate to the dtype. """ - b = bitstring.Bits() + b = object.__new__(bitstring.Bits) self._set_fn(b, value) if self.bitlength is not None and len(b) != self.bitlength: raise ValueError(f"Dtype has a length of {self.bitlength} bits, but value '{value}' has {len(b)} bits.") return b - def parse(self, b: BitsType, /) -> Any: - """Parse a bitstring to find its value. + def unpack(self, b: BitsType, /) -> Any: + """Unpack a bitstring to find its value. The b parameter should be a bitstring of the appropriate length, or an object that can be converted to a bitstring.""" b = bitstring.Bits._create_from_bitstype(b) - return self._get_fn(bitstring.Bits(b)) + return self._get_fn(b) def __str__(self) -> str: if self._scale is not None: @@ -208,7 +209,7 @@ def __repr__(self) -> str: except ValueError: scale_str = f', scale={self._scale}' else: - power_of_two = e8m0.uint - 127 + power_of_two = e8m0.u - 127 if power_of_two in [0, 1]: scale_str = f', scale={self._scale}' else: @@ -217,12 +218,12 @@ def __repr__(self) -> str: def __eq__(self, other: Any) -> bool: if isinstance(other, Dtype): - return self._name == other._name and self._length == other._length + return self._name == other._name and self._length == other._length and self._scale == other._scale return False class AllowedLengths: - def __init__(self, value: Tuple[int, ...] = tuple()) -> None: + def __init__(self, value: tuple[int, ...] = tuple()) -> None: if len(value) >= 3 and value[-1] is Ellipsis: step = value[1] - value[0] for i in range(1, len(value) - 1): @@ -241,7 +242,9 @@ def __contains__(self, other: Any) -> bool: if not self.values: return True if self.values[-1] is Ellipsis: - return (other - self.values[0]) % (self.values[1] - self.values[0]) == 0 + start = self.values[0] + step = self.values[1] - self.values[0] + return other >= start and (other - start) % step == 0 return other in self.values def only_one_value(self) -> bool: @@ -253,7 +256,8 @@ class DtypeDefinition: Not (yet) part of the public interface.""" def __init__(self, name: str, set_fn, get_fn, return_type: Any = Any, is_signed: bool = False, bitlength2chars_fn=None, - variable_length: bool = False, allowed_lengths: Tuple[int, ...] = tuple(), multiplier: int = 1, description: str = ''): + variable_length: bool = False, allowed_lengths: tuple[int, ...] = tuple(), multiplier: int = 1, + description: str = '', read_fn=None): # Consistency checks if int(multiplier) != multiplier or multiplier <= 0: @@ -290,14 +294,28 @@ def allowed_length_checked_get_fn(bs): # Create a reading function from the get_fn. if not self.variable_length: - if self.allowed_lengths.only_one_value(): - def read_fn(bs, start): - return self.get_fn(bs[start:start + self.allowed_lengths.values[0]]) + if read_fn is not None: + direct_read_fn = read_fn + if self.allowed_lengths.only_one_value(): + def read_fn(bs, start): + length = self.allowed_lengths.values[0] + if len(bs) < start + length: + raise bitstring.ReadError(f"Needed a length of at least {length} bits, but only {len(bs) - start} bits were available.") + return direct_read_fn(bs, start, length) + else: + def read_fn(bs, start, length): + if len(bs) < start + length: + raise bitstring.ReadError(f"Needed a length of at least {length} bits, but only {len(bs) - start} bits were available.") + return direct_read_fn(bs, start, length) else: - def read_fn(bs, start, length): - if len(bs) < start + length: - raise bitstring.ReadError(f"Needed a length of at least {length} bits, but only {len(bs) - start} bits were available.") - return self.get_fn(bs[start:start + length]) + if self.allowed_lengths.only_one_value(): + def read_fn(bs, start): + return self.get_fn(bs[start:start + self.allowed_lengths.values[0]]) + else: + def read_fn(bs, start, length): + if len(bs) < start + length: + raise bitstring.ReadError(f"Needed a length of at least {length} bits, but only {len(bs) - start} bits were available.") + return self.get_fn(bs[start:start + length]) self.read_fn = read_fn else: # We only find out the length when we read/get. @@ -317,7 +335,7 @@ def read_fn(bs, start): self.read_fn = read_fn self.bitlength2chars_fn = bitlength2chars_fn - def get_dtype(self, length: Optional[int] = None, scale: Union[None, float, int] = None) -> Dtype: + def get_dtype(self, length: int | None = None, scale: None | float | int = None) -> Dtype: if self.allowed_lengths: if length is None: if self.allowed_lengths.only_one_value(): @@ -345,13 +363,13 @@ def __repr__(self) -> str: class Register: """A singleton class that holds all the DtypeDefinitions. Not (yet) part of the public interface.""" - _instance: Optional[Register] = None - names: Dict[str, DtypeDefinition] = {} + _instance: Register | None = None + names: dict[str, DtypeDefinition] = {} def __new__(cls) -> Register: # Singleton. Only one Register instance can ever exist. if cls._instance is None: - cls._instance = super(Register, cls).__new__(cls) + cls._instance = super().__new__(cls) return cls._instance @classmethod @@ -372,7 +390,7 @@ def add_dtype_alias(cls, name: str, alias: str): setattr(bitstring.bitarray_.BitArray, alias, property(fget=definition.get_fn, fset=definition.set_fn, doc=f"An alias for '{name}'. Read and write.")) @classmethod - def get_dtype(cls, name: str, length: Optional[int], scale: Union[None, float, int] = None) -> Dtype: + def get_dtype(cls, name: str, length: int | None, scale: None | float | int = None) -> Dtype: try: definition = cls.names[name] except KeyError: diff --git a/bitstring/exceptions.py b/bitstring/exceptions.py index 374d3936..d6525ba1 100644 --- a/bitstring/exceptions.py +++ b/bitstring/exceptions.py @@ -1,4 +1,3 @@ - class Error(Exception): """Base class for errors in the bitstring module.""" diff --git a/bitstring/fp8.py b/bitstring/fp8.py index 575dbbb4..8b1e6aea 100644 --- a/bitstring/fp8.py +++ b/bitstring/fp8.py @@ -7,9 +7,9 @@ import struct import zlib import array -import bitarray from bitstring.luts import binary8_luts_compressed import math +import tibs class Binary8Format: @@ -66,17 +66,17 @@ def createLUT_for_binary8_to_float(self): """Create a LUT to convert an int in range 0-255 representing a float8 into a Python float""" i2f = [] for i in range(256): - b = bitarray.util.int2ba(i, length=8, endian='big', signed=False) + b = tibs.Tibs.from_u(i, 8) sign = b[0] - exponent = bitarray.util.ba2int(b[1:1 + self.exp_bits]) + exponent = b.to_u(1, 1 + self.exp_bits) significand = b[1 + self.exp_bits:] if exponent == 0: - significand = bitarray.bitarray('0') + significand + significand = [0] + significand exponent = -self.bias + 1 else: - significand = bitarray.bitarray('1') + significand + significand = [1] + significand exponent -= self.bias - f = float(bitarray.util.ba2int(significand)) / (2.0 ** (7 - self.exp_bits)) + f = float(significand.to_u()) / (2.0 ** (7 - self.exp_bits)) f *= 2 ** exponent i2f.append(f if not sign else -f) # One special case for minus zero diff --git a/bitstring/helpers.py b/bitstring/helpers.py index 3cf3d4f5..dfe8e763 100644 --- a/bitstring/helpers.py +++ b/bitstring/helpers.py @@ -1,7 +1,6 @@ -from typing import Union, Tuple -def _indices(s: slice, length: int) -> Tuple[int, Union[int, None], int]: +def _indices(s: slice, length: int) -> tuple[int, int | None, int]: """A better implementation of slice.indices such that a slice made from [start:stop:step] will actually equal the original slice.""" if s.step is None or s.step > 0: @@ -12,24 +11,6 @@ def _indices(s: slice, length: int) -> Tuple[int, Union[int, None], int]: stop = None return start, stop, step -def offset_slice_indices_lsb0(key: slice, length: int) -> slice: - start, stop, step = _indices(key, length) - if step is not None and step < 0: - if stop is None: - new_start = start + 1 - new_stop = None - else: - first_element = start - last_element = start + ((stop + 1 - start) // step) * step - new_start = length - last_element - new_stop = length - first_element - 1 - else: - first_element = start - # The last element will usually be stop - 1, but needs to be adjusted if step != 1. - last_element = start + ((stop - 1 - start) // step) * step - new_start = length - last_element - 1 - new_stop = length - first_element - return slice(new_start, new_stop, key.step) def tidy_input_string(s: str) -> str: """Return string made lowercase and with all whitespace and underscores removed.""" diff --git a/bitstring/methods.py b/bitstring/methods.py index c224dc9a..7ca5dc44 100644 --- a/bitstring/methods.py +++ b/bitstring/methods.py @@ -1,36 +1,33 @@ from __future__ import annotations import bitstring -from bitstring.bitstream import BitStream +from bitstring.bits import Bits from bitstring.utils import tokenparser from bitstring.exceptions import CreationError -from typing import Union, List +import bitstring.bitstore_helpers as helpers -MutableBitStore = bitstring.bitstore.MutableBitStore -helpers = bitstring.bitstore_helpers -common_helpers = bitstring.bitstore_common_helpers +ConstBitStore = bitstring.bitstore.ConstBitStore - -def pack(fmt: Union[str, List[str]], *values, **kwargs) -> BitStream: - """Pack the values according to the format string and return a new BitStream. +def pack(fmt: str | list[str], *values, **kwargs) -> Bits: + """Pack the values according to the format string and return a new Bits object. fmt -- A single string or a list of strings with comma separated tokens - describing how to create the BitStream. + describing how to create the Bits. values -- Zero or more values to pack according to the format. kwargs -- A dictionary or keyword-value pairs - the keywords used in the format string will be replaced with their given value. - Token examples: 'int:12' : 12 bits as a signed integer - 'uint:8' : 8 bits as an unsigned integer - 'float:64' : 8 bytes as a big-endian float - 'intbe:16' : 2 bytes as a big-endian signed integer - 'uintbe:16' : 2 bytes as a big-endian unsigned integer - 'intle:32' : 4 bytes as a little-endian signed integer - 'uintle:32' : 4 bytes as a little-endian unsigned integer - 'floatle:64': 8 bytes as a little-endian float - 'intne:24' : 3 bytes as a native-endian signed integer - 'uintne:24' : 3 bytes as a native-endian unsigned integer - 'floatne:32': 4 bytes as a native-endian float + Token examples: 'i:12' : 12 bits as a signed integer + 'u:8' : 8 bits as an unsigned integer + 'f:64' : 8 bytes as a big-endian float + 'ibe:16' : 2 bytes as a big-endian signed integer + 'ube:16' : 2 bytes as a big-endian unsigned integer + 'ile:32' : 4 bytes as a little-endian signed integer + 'ule:32' : 4 bytes as a little-endian unsigned integer + 'fle:64' : 8 bytes as a little-endian float + 'ine:24' : 3 bytes as a native-endian signed integer + 'une:24' : 3 bytes as a native-endian unsigned integer + 'fne:32' : 4 bytes as a native-endian float 'hex:80' : 80 bits as a hex string 'oct:9' : 9 bits as an octal string 'bin:1' : single bit binary string @@ -41,9 +38,9 @@ def pack(fmt: Union[str, List[str]], *values, **kwargs) -> BitStream: 'bool' : 1 bit as a bool 'pad:3' : 3 zero bits as padding - >>> s = pack('uint:12, bits', 100, '0xffe') + >>> s = pack('u:12, bits', 100, '0xffe') >>> t = pack(['bits', 'bin:3'], s, '111') - >>> u = pack('uint:8=a, uint:8=b, uint:55=a', a=6, b=44) + >>> u = pack('u:8=a, u:8=b, u:55=a', a=6, b=44) """ tokens = [] @@ -56,7 +53,7 @@ def pack(fmt: Union[str, List[str]], *values, **kwargs) -> BitStream: except ValueError as e: raise CreationError(*e.args) value_iter = iter(values) - bsl: List[BitStore] = [] + bsl: list[ConstBitStore] = [] try: for name, length, value in tokens: # If the value is in the kwd dictionary then it takes precedence. @@ -65,20 +62,14 @@ def pack(fmt: Union[str, List[str]], *values, **kwargs) -> BitStream: length = kwargs.get(length, length) # Also if we just have a dictionary name then we want to use it if name in kwargs and length is None and value is None: - bsl.append(BitStream(kwargs[name])._bitstore) + bsl.append(Bits(kwargs[name])._bitstore) continue if length is not None: length = int(length) if value is None and name != 'pad': # Take the next value from the ones provided value = next(value_iter) - if name == 'bits': - value = bitstring.bits.Bits(value) - if length is not None and length != len(value): - raise CreationError(f"Token with length {length} packed with value of length {len(value)}.") - bsl.append(value._bitstore) - continue - bsl.append(common_helpers.bitstore_from_token(name, length, value)) + bsl.append(helpers.bitstore_from_token(name, length, value)) except StopIteration: raise CreationError(f"Not enough parameters present to pack according to the " f"format. {len(tokens)} values are needed.") @@ -87,11 +78,8 @@ def pack(fmt: Union[str, List[str]], *values, **kwargs) -> BitStream: next(value_iter) except StopIteration: # Good, we've used up all the *values. - s = BitStream() - if bitstring.options.lsb0: - bsl.reverse() - for b in bsl: - s._bitstore += b + s = object.__new__(Bits) + s._bitstore = ConstBitStore.join(bsl) return s raise CreationError(f"Too many parameters present to pack according to the format. Only {len(tokens)} values were expected.") diff --git a/bitstring/mxfp.py b/bitstring/mxfp.py index d675c359..601a8039 100644 --- a/bitstring/mxfp.py +++ b/bitstring/mxfp.py @@ -1,13 +1,12 @@ import array import math import struct -import bitarray +import tibs from bitstring.luts import mxfp_luts_compressed import zlib -from typing import Optional -def round_to_nearest_ties_to_even(lut_int_to_float, lower: int, f: float) -> Optional[int]: +def round_to_nearest_ties_to_even(lut_int_to_float, lower: int, f: float) -> int | None: upper = lower + 1 # Special case for LUTs without a negative zero. lower_float = 0.0 if lower == 128 else lut_int_to_float[lower] @@ -127,17 +126,17 @@ def createLUT_for_int_to_float(self) -> array.array: i2f = [] length = 1 + self.exp_bits + self.mantissa_bits for i in range(1 << length): - b = bitarray.util.int2ba(i, length=length, endian='big', signed=False) + b = tibs.Tibs.from_u(i, length) sign = b[0] - exponent = bitarray.util.ba2int(b[1:1 + self.exp_bits]) + exponent = b.to_u(1, 1 + self.exp_bits) significand = b[1 + self.exp_bits:] if exponent == 0: - significand = bitarray.bitarray('0') + significand + significand = [0] + significand exponent = -self.bias + 1 else: - significand = bitarray.bitarray('1') + significand + significand = [1] + significand exponent -= self.bias - f = float(bitarray.util.ba2int(significand)) / (2.0 ** self.mantissa_bits) + f = float(significand.to_u()) / (2.0 ** self.mantissa_bits) f *= 2 ** exponent if length == 8: # Some special cases diff --git a/bitstring/reader.py b/bitstring/reader.py new file mode 100644 index 00000000..d878515e --- /dev/null +++ b/bitstring/reader.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import numbers +from typing import Any, overload + +import bitstring +from bitstring.bits import Bits, BitsType +from bitstring.dtypes import Dtype + + +class Reader: + """Wrap a Bits or BitArray object with a bit position for sequential reading.""" + + __slots__ = ("_bits", "_pos") + + def __init__(self, bits: Bits, pos: int = 0) -> None: + self.bits = bits + self.pos = pos + + @staticmethod + def _validate_bits(value: Bits) -> None: + if isinstance(value, Bits): + return + help_ = "" + if isinstance(value, str): + help_ = " Perhaps use Bits.from_string() or Bits(...)?" + elif isinstance(value, (bytes, bytearray, memoryview)): + help_ = " Perhaps use Bits(bytes=...)?" + raise TypeError(f"Reader should be initialised with a Bits or BitArray object, " + f"but received a {type(value).__name__}.{help_}") + + @property + def bits(self) -> Bits: + """The wrapped Bits or BitArray object.""" + return self._bits + + @bits.setter + def bits(self, value: Bits) -> None: + Reader._validate_bits(value) + self._bits = value + + @property + def pos(self) -> int: + """The current bit position.""" + return self._pos + + @pos.setter + def pos(self, value: int) -> None: + self._pos = int(value) + + @property + def bitpos(self) -> int: + """An alias for pos.""" + return self._pos + + @bitpos.setter + def bitpos(self, value: int) -> None: + self._pos = int(value) + + @property + def bytepos(self) -> int: + """The current byte position. Requires the bit position to be byte aligned.""" + if self._pos % 8: + raise bitstring.ByteAlignError("Not byte aligned when using bytepos property.") + return self._pos // 8 + + @bytepos.setter + def bytepos(self, value: int) -> None: + self._pos = int(value) * 8 + + def _ensure_valid_pos(self) -> None: + if not 0 <= self._pos <= len(self._bits): + raise ValueError(f"Invalid bit position {self._pos} for bitstring of length {len(self._bits)}.") + + @staticmethod + def _dtype_with_length(dtype: Dtype, items: int) -> Dtype: + return Dtype(dtype.name, items, dtype.scale) + + @overload + def read(self, fmt: int) -> Bits: + ... + + @overload + def read(self, fmt: str) -> Any: + ... + + def read(self, fmt: int | str | Dtype) -> int | float | str | Bits | bool | bytes | None: + """Read from the current bit position and interpret according to fmt.""" + old_pos = self._pos + try: + self._ensure_valid_pos() + if isinstance(fmt, numbers.Integral): + if fmt < 0: + raise ValueError("Cannot read negative amount.") + if fmt > len(self._bits) - self._pos: + raise bitstring.ReadError( + f"Cannot read {fmt} bits, only {len(self._bits) - self._pos} available.") + value = self._bits._slice(self._pos, self._pos + int(fmt)) + self._pos += int(fmt) + return value + + dtype = Dtype(fmt) + if dtype.bitlength is None and not dtype.variable_length: + bitlength = len(self._bits) - self._pos + items, remainder = divmod(bitlength, dtype.bits_per_item) + if remainder != 0: + raise ValueError( + f"The '{dtype.name}' type must have a bit length that is a multiple of " + f"{dtype.bits_per_item} so cannot be read from the {bitlength} bits that are available.") + dtype = Reader._dtype_with_length(dtype, items) + if dtype.bitlength is not None: + value = dtype.read_fn(self._bits, self._pos) + self._pos += dtype.bitlength + else: + value, self._pos = dtype.read_fn(self._bits, self._pos) + if self._pos > len(self._bits): + raise bitstring.ReadError( + f"Reading off end of bitstring with fmt '{fmt}'. Only {len(self._bits) - old_pos} bits available.") + return value + except Exception: + self._pos = old_pos + raise + + def read_list(self, fmt: str | list[int | str | Dtype], **kwargs) \ + -> list[int | float | str | Bits | bool | bytes | None]: + """Read and interpret one or more format tokens from the current bit position.""" + old_pos = self._pos + try: + self._ensure_valid_pos() + value, self._pos = self._bits._readlist(fmt, self._pos, **kwargs) + if self._pos > len(self._bits): + raise bitstring.ReadError( + f"Reading off end of bitstring. Only {len(self._bits) - old_pos} bits available.") + return value + except Exception: + self._pos = old_pos + raise + + def readlist(self, fmt: str | list[int | str | Dtype], **kwargs) \ + -> list[int | float | str | Bits | bool | bytes | None]: + """Compatibility alias for :meth:`read_list`.""" + return self.read_list(fmt, **kwargs) + + @overload + def peek(self, fmt: int) -> Bits: + ... + + @overload + def peek(self, fmt: str) -> Any: + ... + + def peek(self, fmt: int | str | Dtype) -> int | float | str | Bits | bool | bytes | None: + """Read from the current bit position without changing the position.""" + old_pos = self._pos + try: + return self.read(fmt) + finally: + self._pos = old_pos + + def peek_list(self, fmt: str | list[int | str | Dtype], **kwargs) \ + -> list[int | float | str | Bits | bool | bytes | None]: + """Read one or more format tokens without changing the position.""" + old_pos = self._pos + try: + return self.read_list(fmt, **kwargs) + finally: + self._pos = old_pos + + def peeklist(self, fmt: str | list[int | str | Dtype], **kwargs) \ + -> list[int | float | str | Bits | bool | bytes | None]: + """Compatibility alias for :meth:`peek_list`.""" + return self.peek_list(fmt, **kwargs) + + def read_to(self, bs: BitsType, /, *, bytealigned: bool | None = None) -> Bits: + """Read up to and including the next occurrence of bs.""" + if isinstance(bs, numbers.Integral): + raise ValueError("Integers cannot be searched for") + old_pos = self._pos + try: + self._ensure_valid_pos() + bs = Bits._create_from_bitstype(bs) + p = self.find(bs, start=self._pos, bytealigned=bytealigned) + if p is None: + raise bitstring.ReadError("Substring not found") + self._pos = p + len(bs) + return self._bits._slice(old_pos, self._pos) + except Exception: + self._pos = old_pos + raise + + def readto(self, bs: BitsType, /, *, bytealigned: bool | None = None) -> Bits: + """Compatibility alias for :meth:`read_to`.""" + return self.read_to(bs, bytealigned=bytealigned) + + def byte_align(self) -> int: + """Align to the next byte boundary and return the number of skipped bits.""" + self._ensure_valid_pos() + skipped = (8 - (self._pos % 8)) % 8 + if self._pos + skipped > len(self._bits): + raise ValueError("Cannot seek past the end of the data.") + self._pos += skipped + return skipped + + def bytealign(self) -> int: + """Compatibility alias for :meth:`byte_align`.""" + return self.byte_align() + + def find(self, bs: BitsType, /, *, start: int | None = None, end: int | None = None, + bytealigned: bool | None = None) -> int | None: + """Find a bitstring and set pos to the match position if found.""" + p = self._bits.find(bs, start=start, end=end, bytealigned=bytealigned) + if p is not None: + self._pos = p + return p + + def rfind(self, bs: BitsType, /, *, start: int | None = None, end: int | None = None, + bytealigned: bool | None = None) -> int | None: + """Find a bitstring from the end and set pos to the match position if found.""" + p = self._bits.rfind(bs, start=start, end=end, bytealigned=bytealigned) + if p is not None: + self._pos = p + return p + + def __len__(self) -> int: + return len(self._bits) + + def __repr__(self) -> str: + return f"Reader(<{self._bits.__class__.__name__} of length {len(self._bits)} bits>, pos={self._pos})" + + __str__ = __repr__ diff --git a/bitstring/utils.py b/bitstring/utils.py index dec6180e..4eb64dab 100644 --- a/bitstring/utils.py +++ b/bitstring/utils.py @@ -2,7 +2,7 @@ import functools import re -from typing import Tuple, List, Optional, Pattern, Dict, Union, Match +from re import Pattern, Match # A token name followed by optional : then an integer number @@ -32,34 +32,34 @@ # These replicate the struct.pack codes # Big-endian -REPLACEMENTS_BE: Dict[str, str] = {'b': 'int8', 'B': 'uint8', - 'h': 'intbe16', 'H': 'uintbe16', - 'l': 'intbe32', 'L': 'uintbe32', - 'i': 'intbe32', 'I': 'uintbe32', - 'q': 'intbe64', 'Q': 'uintbe64', - 'e': 'floatbe16', 'f': 'floatbe32', 'd': 'floatbe64'} +REPLACEMENTS_BE: dict[str, str] = {'b': 'i8', 'B': 'u8', + 'h': 'ibe16', 'H': 'ube16', + 'l': 'ibe32', 'L': 'ube32', + 'i': 'ibe32', 'I': 'ube32', + 'q': 'ibe64', 'Q': 'ube64', + 'e': 'f16', 'f': 'f32', 'd': 'f64'} # Little-endian -REPLACEMENTS_LE: Dict[str, str] = {'b': 'int8', 'B': 'uint8', - 'h': 'intle16', 'H': 'uintle16', - 'l': 'intle32', 'L': 'uintle32', - 'i': 'intle32', 'I': 'uintle32', - 'q': 'intle64', 'Q': 'uintle64', - 'e': 'floatle16', 'f': 'floatle32', 'd': 'floatle64'} +REPLACEMENTS_LE: dict[str, str] = {'b': 'i8', 'B': 'u8', + 'h': 'ile16', 'H': 'ule16', + 'l': 'ile32', 'L': 'ule32', + 'i': 'ile32', 'I': 'ule32', + 'q': 'ile64', 'Q': 'ule64', + 'e': 'fle16', 'f': 'fle32', 'd': 'fle64'} # Native-endian -REPLACEMENTS_NE: Dict[str, str] = {'b': 'int8', 'B': 'uint8', - 'h': 'intne16', 'H': 'uintne16', - 'l': 'intne32', 'L': 'uintne32', - 'i': 'intne32', 'I': 'uintne32', - 'q': 'intne64', 'Q': 'uintne64', - 'e': 'floatne16', 'f': 'floatne32', 'd': 'floatne64'} +REPLACEMENTS_NE: dict[str, str] = {'b': 'i8', 'B': 'u8', + 'h': 'ine16', 'H': 'une16', + 'l': 'ine32', 'L': 'une32', + 'i': 'ine32', 'I': 'une32', + 'q': 'ine64', 'Q': 'une64', + 'e': 'fne16', 'f': 'fne32', 'd': 'fne64'} # Size in bytes of all the pack codes. -PACK_CODE_SIZE: Dict[str, int] = {'b': 1, 'B': 1, 'h': 2, 'H': 2, 'l': 4, 'L': 4, 'i': 4, 'I': 4, +PACK_CODE_SIZE: dict[str, int] = {'b': 1, 'B': 1, 'h': 2, 'H': 2, 'l': 4, 'L': 4, 'i': 4, 'I': 4, 'q': 8, 'Q': 8, 'e': 2, 'f': 4, 'd': 8} -def structparser(m: Match[str]) -> List[str]: +def structparser(m: Match[str]) -> list[str]: """Parse struct-like format string token into sub-token list.""" endian = m.group('endian') # Split the format string into a list of 'q', '4h' etc. @@ -79,7 +79,7 @@ def structparser(m: Match[str]) -> List[str]: @functools.lru_cache(CACHE_SIZE) -def parse_name_length_token(fmt: str, **kwargs) -> Tuple[str, Optional[int]]: +def parse_name_length_token(fmt: str, **kwargs) -> tuple[str, int | None]: # Any single token with just a name and length if m2 := NAME_INT_RE.match(fmt): name = m2.group(1) @@ -100,7 +100,7 @@ def parse_name_length_token(fmt: str, **kwargs) -> Tuple[str, Optional[int]]: @functools.lru_cache(CACHE_SIZE) -def parse_single_struct_token(fmt: str) -> Optional[Tuple[str, Optional[int]]]: +def parse_single_struct_token(fmt: str) -> tuple[str, int | None] | None: if m := SINGLE_STRUCT_PACK_RE.match(fmt): endian = m.group('endian') f = m.group('fmt') @@ -117,7 +117,7 @@ def parse_single_struct_token(fmt: str) -> Optional[Tuple[str, Optional[int]]]: @functools.lru_cache(CACHE_SIZE) -def parse_single_token(token: str) -> Tuple[str, str, Optional[str]]: +def parse_single_token(token: str) -> tuple[str, str, str | None]: if (equals_pos := token.find('=')) == -1: value = None else: @@ -140,7 +140,7 @@ def parse_single_token(token: str) -> Tuple[str, str, Optional[str]]: @functools.lru_cache(CACHE_SIZE) -def preprocess_tokens(fmt: str) -> List[str]: +def preprocess_tokens(fmt: str) -> list[str]: # Remove whitespace and expand brackets fmt = expand_brackets(''.join(fmt.split())) @@ -156,6 +156,8 @@ def preprocess_tokens(fmt: str) -> List[str]: factor = 1 if m := MULTIPLICATIVE_RE.match(meta_token): factor = int(m.group('factor')) + if factor < 0: + raise ValueError(f"Negative multiplicative factors are not allowed: '{meta_token}'.") meta_token = m.group('token') # Parse struct-like format into sub-tokens or treat as single token @@ -167,12 +169,12 @@ def preprocess_tokens(fmt: str) -> List[str]: @functools.lru_cache(CACHE_SIZE) -def tokenparser(fmt: str, keys: Tuple[str, ...] = ()) -> \ - Tuple[bool, List[Tuple[str, Union[int, str, None], Optional[str]]]]: +def tokenparser(fmt: str, keys: tuple[str, ...] = ()) -> \ + tuple[bool, list[tuple[str, int | str | None, str | None]]]: """Divide the format string into tokens and parse them. Return stretchy token and list of [initialiser, length, value] - initialiser is one of: hex, oct, bin, uint, int, se, ue, 0x, 0o, 0b etc. + initialiser is one of: hex, oct, bin, u, i, se, ue, 0x, 0o, 0b etc. length is None if not known, as is value. If the token is in the keyword dictionary (keys) then it counts as a @@ -183,7 +185,7 @@ def tokenparser(fmt: str, keys: Tuple[str, ...] = ()) -> \ """ tokens = preprocess_tokens(fmt) stretchy_token = False - ret_vals: List[Tuple[str, Union[str, int, None], Optional[str]]] = [] + ret_vals: list[tuple[str, str | int | None, str | None]] = [] for token in tokens: if keys and token in keys: # Don't bother parsing it, it's a keyword argument @@ -235,7 +237,10 @@ def expand_brackets(s: str) -> str: if m: factor = int(m.group('factor')) matchstart = m.start('factor') - s = s[0:matchstart] + (factor - 1) * (s[start + 1:p] + ',') + s[start + 1:p] + s[p + 1:] + if factor == 0: + s = s[0:matchstart] + s[p + 1:] + else: + s = s[0:matchstart] + (factor - 1) * (s[start + 1:p] + ',') + s[start + 1:p] + s[p + 1:] else: raise ValueError(f"Failed to parse '{s}'.") return s diff --git a/doc/appendices.rst b/doc/appendices.rst index 1ca03f0d..172f7ac7 100644 --- a/doc/appendices.rst +++ b/doc/appendices.rst @@ -12,3 +12,4 @@ This section contains supplementary information about the library, including det exotic_floats exp-golomb optimisation + upgrading_to_version_5 diff --git a/doc/array.rst b/doc/array.rst index 0332b6c3..5fb05772 100644 --- a/doc/array.rst +++ b/doc/array.rst @@ -10,11 +10,11 @@ Array This can be any format which has a fixed length. See :ref:`format_tokens` and :ref:`compact_format` for details on allowed dtype strings, noting that only formats with well defined bit lengths are allowed. - The `inititalizer` will typically be an iterable such as a list, but can also be many other things including an open binary file, a bytes or bytearray object, another ``bitstring.Array`` or an ``array.array``. + The `initializer` will typically be an iterable such as a list, but can also be many other things including an open binary file, a bytes or bytearray object, another ``bitstring.Array`` or an ``array.array``. It can also be an integer, in which case the ``Array`` will be zero-initialised with that many items. :: >>> bitstring.Array('i4', 8) - Array('int4', [0, 0, 0, 0, 0, 0, 0, 0]) + Array('i4', [0, 0, 0, 0, 0, 0, 0, 0]) The `trailing_bits` typically isn't used in construction, and specifies bits left over after interpreting the stored binary data according to the data type `dtype`. @@ -28,13 +28,13 @@ The ``bitstring.Array`` type is meant as a more flexible version of the standard x = array.array('f', [1.0, 2.0, 3.14]) y = bitstring.Array('=f', [1.0, 2.0, 3.14]) - assert x.tobytes() == y.tobytes() + assert x.to_bytes() == y.to_bytes() This example packs three 32-bit floats into objects using both libraries. The only difference is the explicit native endianness for the format string of the bitstring version. The bitstring Array's advantage lies in the way that any fixed-length bitstring format can be used instead of just the dozen or so typecodes supported by the ``array`` module. -For example ``'uint4'``, ``'bfloat'`` or ``'hex12'`` can be used, and the endianness of multi-byte dtypes can be properly specified. +For example ``'u4'``, ``'bfloat'`` or ``'hex12'`` can be used, and the endianness of multi-byte dtypes can be properly specified. Each element in the ``Array`` must then be something that makes sense for the ``dtype``. Some examples will help illustrate:: @@ -42,17 +42,17 @@ Some examples will help illustrate:: from bitstring import Array # Each unsigned int is stored in 4 bits - a = Array('uint4', [0, 5, 5, 3, 2]) + a = Array('u4', [0, 5, 5, 3, 2]) # Convert and store floats in 8 bits each b = Array('p3binary', [-56.0, 0.123, 99.6]) # Each element is a 7 bit signed integer - c = Array('int7', [-3, 0, 120]) + c = Array('i7', [-3, 0, 120]) You can then access and modify the ``Array`` with the usual notation:: - a[1:4] # Array('uint4', [5, 5, 3]) + a[1:4] # Array('u4', [5, 5, 3]) b[0] # -56.0 c[-1] # 120 @@ -62,27 +62,27 @@ You can then access and modify the ``Array`` with the usual notation:: Conversion between ``Array`` types can be done using the :meth:`astype` method. If elements of the old array don't fit or don't make sense in the new array then the relevant exceptions will be raised. :: - >>> x = Array('float64', [89.3, 1e34, -0.00000001, 34]) - >>> y = x.astype('float16') + >>> x = Array('f64', [89.3, 1e34, -0.00000001, 34]) + >>> y = x.astype('f16') >>> y - Array('float16', [89.3125, inf, -0.0, 34.0]) + Array('f16', [89.3125, inf, -0.0, 34.0]) >>> y = y.astype('p4binary') >>> y Array('p4binary', [88.0, 240.0, 0.0, 32.0]) - >>> y.astype('uint8') - Array('uint8', [88, 240, 0, 32]) - >>> y.astype('uint7') + >>> y.astype('u8') + Array('u8', [88, 240, 0, 32]) + >>> y.astype('u7') bitstring.CreationError: 240 is too large an unsigned integer for a bitstring of length 7. The allowed range is [0, 127]. You can also reinterpret the data by changing the :attr:`dtype` property directly. This will not copy any data but will cause the current data to be shown differently. :: - >>> x = Array('int16', [-5, 100, -4]) + >>> x = Array('i16', [-5, 100, -4]) >>> x - Array('int16', [-5, 100, -4]) - >>> x.dtype = 'int8' + Array('i16', [-5, 100, -4]) + >>> x.dtype = 'i8' >>> x - Array('int8', [-1, -5, 0, 100, -1, -4]) + Array('i8', [-1, -5, 0, 100, -1, -4]) The data for the array is stored internally as a :class:`BitArray` object. @@ -95,7 +95,7 @@ Typically :attr:`trailing_bits` will be an empty :class:`BitArray` but if you ch Some methods, such as :meth:`~Array.append` and :meth:`~Array.extend` will raise an exception if used when :attr:`trailing_bits` is not empty, as it not clear how these should behave in this case. You can however still use :meth:`~Array.insert` which will always leave the :attr:`trailing_bits` unchanged. -The :attr:`dtype` string can be a type code such as ``'>H'`` or ``'=d'`` but it can also be a string defining any format which has a fixed-length in bits, for example ``'int12'``, ``'bfloat'``, ``'bytes5'`` or ``'bool'``. +The :attr:`dtype` string can be a type code such as ``'>H'`` or ``'=d'`` but it can also be a string defining any format which has a fixed-length in bits, for example ``'i12'``, ``'bfloat'``, ``'bytes5'`` or ``'bool'``. Note that the typecodes must include an endianness character to give the byte ordering. This is more like the ``struct`` module typecodes, and is different to the ``array.array`` typecodes which are always native-endian. @@ -105,17 +105,17 @@ The correspondence between the big-endian type codes and bitstring dtype strings ========= =================== Type code bitstring dtype ========= =================== -``'>b'`` ``'int8'`` -``'>B'`` ``'uint8'`` -``'>h'`` ``'int16'`` -``'>H'`` ``'uint16'`` -``'>l'`` ``'int32'`` -``'>L'`` ``'uint32'`` -``'>q'`` ``'int64'`` -``'>Q'`` ``'uint64'`` -``'>e'`` ``'float16'`` -``'>f'`` ``'float32'`` -``'>d'`` ``'float64'`` +``'>b'`` ``'i8'`` +``'>B'`` ``'u8'`` +``'>h'`` ``'i16'`` +``'>H'`` ``'u16'`` +``'>l'`` ``'i32'`` +``'>L'`` ``'u32'`` +``'>q'`` ``'i64'`` +``'>Q'`` ``'u64'`` +``'>e'`` ``'f16'`` +``'>f'`` ``'f32'`` +``'>d'`` ``'f64'`` ========= =================== The endianness character can be ``'>'`` for big-endian, ``'<'`` for little-endian or ``'='`` for native-endian (``'@'`` can also be used for native-endian). @@ -124,9 +124,9 @@ In the bitstring dtypes the default is big-endian, but you can specify little or ============ ============================= Type code bitstring dtype ============ ============================= -``'>H'`` ``'uint16'`` / ``'uintbe16'`` -``'=H'`` ``'uintne16'`` -``'H'`` ``'u16'`` / ``'ube16'`` +``'=H'`` ``'une16'`` +``'>> a = Array('float64', [-990, 34, 1, 0.25]) + >>> a = Array('f64', [-990, 34, 1, 0.25]) >>> a.data BitArray('0xc08ef0000000000040410000000000003ff00000000000003fd0000000000000') - >>> b = a.astype('float16') + >>> b = a.astype('f16') >>> b.data BitArray('0xe3bc50403c003400') >>> a == b @@ -173,13 +173,13 @@ Methods Raises a ``ValueError`` if the format is not an integer number of bytes long. :: - >>> a = Array('uint32', [100, 1, 999]) + >>> a = Array('u32', [100, 1, 999]) >>> a.byteswap() >>> a - Array('uint32', [1677721600, 16777216, 3875733504]) - >>> a.dtype = 'uintle32' + Array('u32', [1677721600, 16777216, 3875733504]) + >>> a.dtype = 'ule32' >>> a - Array('uintle32', [100, 1, 999]) + Array('ule32', [100, 1, 999]) .. method:: Array.count(value: float | int | str | bytes) -> int @@ -206,9 +206,9 @@ Methods >>> a.equals(b) False - To compare only the values contained in the Array, extract them using :meth:`~Array.tolist` first:: + To compare only the values contained in the Array, extract them using :meth:`~Array.to_list` first:: - >>> a.tolist() == b.tolist() + >>> a.to_list() == b.to_list() True Note that the ``==`` operator will perform an element-wise equality check and return a new ``Array`` of dtype ``'bool'`` (or raise an exception). @@ -223,15 +223,17 @@ Methods The `iterable` can be another ``Array`` or an ``array.array``, but only if the dtype is the same. :: - >>> a = Array('int5', [-5, 0, 10]) + >>> a = Array('i5', [-5, 0, 10]) >>> a.extend([3, 2, 1]) >>> a.extend(a[0:3] // 5) >>> a - Array('int5', [-5, 0, 10, 3, 2, 1, -1, 0, 2]) + Array('i5', [-5, 0, 10, 3, 2, 1, -1, 0, 2]) -.. method:: Array.fromfile(f: BinaryIO, n: int | None = None) -> None +.. method:: Array.from_file(f: BinaryIO, n: int | None = None) -> None - Append items read from a file object. + Append items read from the file object's current position. + + If *n* is specified then at most that many items will be appended, and an :exc:`EOFError` will be raised if there is not enough data. .. method:: Array.insert(i: int, x: float | int | str | bytes) -> None @@ -249,7 +251,7 @@ Methods If a position isn't specified the final item is returned and removed. :: - >>> Array('bytes3', [b'ABC', b'DEF', b'ZZZ']) + >>> a = Array('bytes3', [b'ABC', b'DEF', b'ZZZ']) >>> a.pop(0) b'ABC' >>> a.pop() @@ -266,7 +268,7 @@ Methods If a `fmt` doesn't have an explicit length, the Array's :attr:`itemsize` will be used. - A pair of comma-separated format strings can also be used - if both formats specify a length they must be the same. For example ``'float, hex16'`` or ``'u4, b4'``. + A pair of comma-separated format strings can also be used - if both formats specify a length they must be the same. For example ``'f, hex16'`` or ``'u4, bin4'``. The output will try to stay within `width` characters per line, but will always output at least one element value. @@ -319,22 +321,22 @@ Methods >>> a Array('>L', [300, 200, 100]) -.. method:: Array.tobytes() -> bytes +.. method:: Array.to_bytes() -> bytes Return Array data as bytes object, padding with zero bits at the end if needed. :: >>> a = Array('i4', [3, -6, 2, -3, 2, -7]) - >>> a.tobytes() + >>> a.to_bytes() b':-)' -.. method:: Array.tofile(f: BinaryIO) -> None +.. method:: Array.to_file(f: BinaryIO) -> None Writes the Array data to the file object *f*, which should have been opened in binary write mode. The data written will be padded at the end with between zero and seven ``0`` bits to make it byte aligned. - The file object remains open so the user must call `.close()` on it once they are finished.:: + The file object remains open so the user must call ``.close()`` on it once they are finished. -.. method:: Array.tolist() -> List[float | int | str | bytes] +.. method:: Array.to_list() -> List[float | int | str | bytes] Return Array items as a list. @@ -365,13 +367,39 @@ For other operators, one of the two input ``Array`` dtypes is used as the output Some examples should help illustrate: -=========== ================ ============ ================ === ================== -**Rule 0** ``'uint8'`` ``<=`` ``'float64'`` → ``'bool'`` -**Rule 1** ``'int32'`` ``+`` ``'float16'`` → ``'float16'`` -**Rule 2** ``'uint20'`` ``//`` ``'int10'`` → ``'int10'`` -**Rule 3** ``'int8'`` ``*`` ``'int16'`` → ``'int16'`` -**Rule 4** ``'float16'`` ``-=`` ``'bfloat'`` → ``'float16'`` -=========== ================ ============ ================ === ================== +.. list-table:: + :widths: 14 16 10 16 8 16 + + * - **Rule 0** + - ``'u8'`` + - ``<=`` + - ``'f64'`` + - → + - ``'bool'`` + * - **Rule 1** + - ``'i32'`` + - ``+`` + - ``'f16'`` + - → + - ``'f16'`` + * - **Rule 2** + - ``'u20'`` + - ``//`` + - ``'i10'`` + - → + - ``'i10'`` + * - **Rule 3** + - ``'i8'`` + - ``*`` + - ``'i16'`` + - → + - ``'i16'`` + * - **Rule 4** + - ``'f16'`` + - ``-=`` + - ``'bfloat'`` + - → + - ``'f16'`` Comparison operators """""""""""""""""""" @@ -482,10 +510,10 @@ Python language operators Return the number of elements in the Array. :: - >>> a = Array('uint20', [1, 2, 3]) + >>> a = Array('u20', [1, 2, 3]) >>> len(a) 3 - >>> a.dtype = 'uint1' + >>> a.dtype = 'u1' >>> len(a) 60 @@ -553,5 +581,3 @@ Properties This will typically be an empty ``BitArray``, but if the ``dtype`` or the ``data`` of an ``Array`` object has been altered after its creation then there may be left-over bits at the end of the data. Note that any methods that append items to the ``Array`` will fail with a ``ValueError`` if there are any trailing bits. - - diff --git a/doc/bitarray.rst b/doc/bitarray.rst index cf86b069..9eb92f78 100644 --- a/doc/bitarray.rst +++ b/doc/bitarray.rst @@ -3,18 +3,31 @@ BitArray ======== -.. class:: BitArray(auto: BitsType | int | None, /, length: int | None = None, offset: int | None = None, **kwargs) +.. class:: BitArray(auto: BitsType | None, /, length: int | None = None, offset: int | None = None, **kwargs) The :class:`Bits` class is the base class for :class:`BitArray` and so (with the exception of :meth:`~Bits.__hash__`) all of its methods are also available for :class:`BitArray` objects. The initialiser is the same as for :class:`Bits`. A :class:`BitArray` is a mutable :class:`Bits`, and so the one thing all of the methods listed here have in common is that they can modify the contents of the bitstring. + The factory methods :meth:`~Bits.from_string`, :meth:`~Bits.from_dtype`, :meth:`~Bits.from_bytes`, :meth:`~Bits.from_bools`, :meth:`~Bits.from_zeros`, :meth:`~Bits.from_ones`, :meth:`~Bits.from_joined`, :meth:`~Bits.from_file` and :meth:`~Bits.from_tibs` are also available and return mutable ``BitArray`` objects when called on ``BitArray``. + ``BitArray.from_tibs`` accepts both ``tibs.Tibs`` and ``tibs.Mutibs`` objects, and always creates a mutable copy. + ---- Methods ------- +.. method:: BitArray.to_bits() -> Bits + + Returns an immutable copy of the bitstring. + +.. method:: BitArray.to_tibs() -> tibs.Tibs + + Returns the data as a ``tibs.Tibs`` instance. + + This is an immutable snapshot of the current :class:`BitArray` data, intended for interoperation with the lower-level ``tibs`` library. + .. method:: BitArray.append(bs: BitsType) -> None Join a :class:`BitArray` to the end of the current :class:`BitArray`. :: @@ -48,9 +61,9 @@ Methods It can also be used to swap the endianness of the whole :class:`BitArray`. :: - >>> s = BitArray('uintle32=1234') + >>> s = BitArray('ule32=1234') >>> s.byteswap() - >>> print(s.uintbe) + >>> print(s.ube) 1234 .. method:: BitArray.clear() -> None @@ -63,15 +76,15 @@ Methods Inserts *bs* at *pos*. - When used with the :class:`BitStream` class the *pos* is optional, and if not present the current bit position will be used. After insertion the property :attr:`~ConstBitStream.pos` will be immediately after the inserted bitstring. :: + :: - >>> s = BitStream('0xccee') + >>> s = BitArray('0xccee') >>> s.insert('0xd', 8) >>> s - BitStream('0xccdee') - >>> s.insert('0x00') + BitArray('0xccdee') + >>> s.insert('0x00', 12) >>> s - BitStream('0xccd00ee') + BitArray('0xccd00ee') .. method:: BitArray.invert(pos: int | Iterable[int] | None = None) -> None @@ -94,14 +107,12 @@ Methods Replaces the contents of the current :class:`BitArray` with *bs* at *pos*. - When used with the :class:`BitStream` class the *pos* is optional, and if not present the current bit position will be used. After insertion the property :attr:`~ConstBitStream.pos` will be immediately after the overwritten bitstring. :: + :: >>> s = BitArray(length=10) >>> s.overwrite('0b111', 3) >>> s BitArray('0b0001110000') - >>> s.pos - 6 .. method:: BitArray.prepend(bs: BitsType) -> None @@ -112,7 +123,7 @@ Methods >>> s BitArray('0b11110') -.. method:: BitArray.replace(old: BitsType, new: BitsType, start: int | None = None, end: int | None = None, count: int | None = None, bytealigned: bool | None = None) -> int +.. method:: BitArray.replace(old: BitsType, new: BitsType, *, start: int | None = None, end: int | None = None, count: int | None = None, bytealigned: bool | None = None) -> int Finds occurrences of *old* and replaces them with *new*. Returns the number of replacements made. @@ -128,7 +139,7 @@ Methods >>> print(s.bin) 0011001111 -.. method:: BitArray.reverse(start: int | None = None, end: int | None = None) -> None +.. method:: BitArray.reverse(*, start: int | None = None, end: int | None = None) -> None Reverses bits in the :class:`BitArray` in-place. @@ -138,11 +149,11 @@ Methods >>> a.reverse() >>> a.bin '101100000' - >>> a.reverse(0, 4) + >>> a.reverse(start=0, end=4) >>> a.bin '110100000' -.. method:: BitArray.rol(bits: int, start: int | None = None, end: int | None = None) -> None +.. method:: BitArray.rol(bits: int, *, start: int | None = None, end: int | None = None) -> None Rotates the contents of the :class:`BitArray` in-place by *bits* bits to the left. @@ -155,7 +166,7 @@ Methods >>> s.bin '00000101' -.. method:: BitArray.ror(bits: int, start: int | None = None, end: int | None = None) -> None +.. method:: BitArray.ror(bits: int, *, start: int | None = None, end: int | None = None) -> None Rotates the contents of the :class:`BitArray` in-place by *bits* bits to the right. @@ -188,29 +199,30 @@ Methods Properties ---------- -Note that the ``bin``, ``oct``, ``hex``, ``int``, ``uint`` and ``float`` properties can all be shortened to their initial letter. -Properties can also have a length in bits appended to them to make properties such as ``u8`` or ``floatle64`` (with the exception of the ``bytes`` property which uses a unit of bytes instead of bits, so ``bytes4`` is 32 bits long). These properties with lengths can be used to quickly create a new bitstring. :: +Note that the ``i``, ``u`` and ``f`` properties are the preferred names for bit-wise big-endian integer and floating point interpretations. +The longer ``int``, ``uint`` and ``float`` names remain as compatibility aliases. +Properties can also have a length in bits appended to them to make properties such as ``u8``, ``hex16`` or ``fle64`` (with the exception of the ``bytes`` property which uses a unit of bytes instead of bits, so ``bytes4`` is 32 bits long). These properties with lengths can be used to quickly create a new bitstring. :: >>> a = BitArray() >>> a.f32 = 17.6 - >>> a.h + >>> a.hex '418ccccd' >>> a.i7 = -1 - >>> a.b + >>> a.bin '1111111' The binary interpretation properties of the :class:`Bits` class all become writable in the :class:`BitArray` class. -For integer types, the properties can have a bit length appended to it such as ``u32`` or ``int5`` to specify the new length of the bitstring. +For integer types, the properties can have a bit length appended to it such as ``u32`` or ``i5`` to specify the new length of the bitstring. Using a length too small to contain the value given will raise a :exc:`CreationError`. When used as a setter without a new length the value must fit into the current length of the :class:`BitArray`, else a :exc:`ValueError` will be raised. :: >>> s = BitArray('0xf3') - >>> s.int + >>> s.i -13 - >>> s.int = 1232 + >>> s.i = 1232 ValueError: int 1232 is too large for a BitArray of length 8. @@ -238,9 +250,10 @@ Special Methods >>> s = BitArray(ue=423) >>> s += BitArray(ue=12) - >>> s.read('ue') + >>> r = Reader(s) + >>> r.read('ue') 423 - >>> s.read('ue') + >>> r.read('ue') 12 .. method:: BitArray.__iand__(bs) @@ -297,4 +310,3 @@ Special Methods >>> s[-12:] = '0xf' >>> print(s) 0x80808f - diff --git a/doc/bits.rst b/doc/bits.rst index 7b2ba49f..72b9576c 100644 --- a/doc/bits.rst +++ b/doc/bits.rst @@ -3,15 +3,15 @@ Bits ==== -The ``Bits`` class is the simplest type in the bitstring module, and represents an immutable sequence of bits. This is the best class to use if you will not need to modify the data after creation and don't need streaming methods. +The ``Bits`` class is the simplest type in the bitstring module, and represents an immutable sequence of bits. This is the best class to use if you will not need to modify the data after creation. -.. class:: Bits(auto: BitsType | int | None, /, length: int | None = None, offset: int | None = None, **kwargs) +.. class:: Bits(auto: BitsType | None, /, length: int | None = None, offset: int | None = None, **kwargs) Creates a new bitstring. - You must specify either no initialiser, just an 'auto' value as the first parameter, or a keyword argument such as ``bytes``, ``bin``, ``hex``, ``oct``, ``uint``, ``int``, ``float``, ``bool`` or ``filename`` (for example) to indicate the data type. + You must specify either no initialiser, just an 'auto' value as the first parameter, or a keyword argument such as ``bytes``, ``bin``, ``hex``, ``oct``, ``u``, ``i``, ``f``, ``bool`` or ``filename`` (for example) to indicate the data type. If no initialiser is given then a zeroed bitstring of ``length`` bits is created. - The initialiser for the :class:`Bits` class is precisely the same as for :class:`BitArray`, :class:`BitStream` and :class:`ConstBitStream`. + The initialiser for the :class:`Bits` class is precisely the same as for :class:`BitArray`. ``offset`` is available when using the ``bytes`` or ``filename`` initialisers. It gives a number of bits to ignore at the start of the bitstring. @@ -24,18 +24,18 @@ The ``Bits`` class is the simplest type in the bitstring module, and represents >>> s1 = Bits(hex='0x934') >>> s2 = Bits(oct='0o4464') >>> s3 = Bits(bin='0b001000110100') - >>> s4 = Bits(int=-1740, length=12) - >>> s5 = Bits(uint=2356, length=12) + >>> s4 = Bits(i=-1740, length=12) + >>> s5 = Bits(u=2356, length=12) >>> s6 = Bits(bytes=b'\x93@', length=12) >>> s1 == s2 == s3 == s4 == s5 == s6 True - See also :ref:`auto_init`, which allows many different types to be used to initialise a bitstring. :: + See also :ref:`auto_init`, which describes the string-based auto initialiser. :: - >>> s = Bits('uint12=32, 0b110') - >>> t = Bits('0o755, ue=12, int:3=-1') + >>> s = Bits('u12=32, 0b110') + >>> t = Bits('0o755, ue=12, i:3=-1') - In the methods below we use ``BitsType`` to indicate that any of the types that can auto initialise can be used. + In the methods below we use ``BitsType`` to indicate that values can be promoted to bitstrings where needed. ---- @@ -50,7 +50,7 @@ Methods *pos* should be an iterable of bit positions. Negative numbers are treated in the same way as slice indices and it will raise an :exc:`IndexError` if ``pos < -len(s)`` or ``pos > len(s)``. It defaults to the whole bitstring. - >>> s = Bits('int15=-1') + >>> s = Bits('i15=-1') >>> s.all(True, [3, 4, 12, 13]) True >>> s.all(1) @@ -76,7 +76,7 @@ Methods Returns a copy of the bitstring. - ``s.copy()`` is equivalent to the shallow copy ``s[:]`` and creates a new copy of the bitstring in memory. + As ``Bits`` is immutable this can return ``self``. For a mutable copy use :meth:`to_bitarray`. .. method:: Bits.count(value: bool) -> int @@ -85,7 +85,7 @@ Methods *value* can be ``True`` or ``False`` or anything that can be cast to a bool, so you could equally use ``1`` or ``0``. - >>> s = BitArray(1000000) + >>> s = BitArray.from_zeros(1000000) >>> s.set(1, [4, 44, 444444]) >>> s.count(1) 3 @@ -96,7 +96,7 @@ Methods Note that if the bitstring is very sparse, as in the example here, it could be quicker to find and count all the set bits with something like ``len(list(s.findall('0b1')))``. For bitstrings with more entropy the ``count`` method will be much quicker than finding. -.. method:: Bits.cut(bits: int, start: int | None = None, end: int | None = None, count: int | None = None) -> Iterator[Bits] +.. method:: Bits.cut(bits: int, *, start: int | None = None, end: int | None = None, count: int | None = None) -> Iterator[Bits] Returns a generator for slices of the bitstring of length *bits*. @@ -109,7 +109,7 @@ Methods 0x43211234 -.. method:: Bits.endswith(bs: BitsType, start: int | None = None, end: int | None = None) -> bool +.. method:: Bits.endswith(bs: BitsType, *, start: int | None = None, end: int | None = None) -> bool Returns ``True`` if the bitstring ends with the sub-string *bs*, otherwise returns ``False``. @@ -122,20 +122,20 @@ Methods False -.. method:: Bits.find(bs: BitsType, start: int | None = None, end: int | None = None, bytealigned: bool | None = None) -> Tuple[int] | Tuple[()] +.. method:: Bits.find(bs: BitsType, *, start: int | None = None, end: int | None = None, bytealigned: bool | None = None) -> int | None - Searches for *bs* in the current bitstring and sets :attr:`~ConstBitStream.pos` to the start of *bs* and returns it in a tuple if found, otherwise it returns an empty tuple. + Searches for *bs* in the current bitstring and returns the start position if found, otherwise it returns ``None``. - The reason for returning the bit position in a tuple is so that it evaluates as True even if the bit position is zero. This allows constructs such as ``if s.find('0xb3'):`` to work as expected. + As bit position zero is a valid result, use ``s.find(...) is not None`` when testing whether a match was found. If *bytealigned* is ``True`` then it will look for *bs* only at byte aligned positions (which is generally much faster than searching for it in every possible bit position). *start* and *end* give the search range and default to the whole bitstring. :: >>> s = Bits('0x0023122') >>> s.find('0b000100', bytealigned=True) - (16,) + 16 -.. method:: Bits.findall(bs: BitsType, start: int | None = None, end: int | None = None, count: int | None = None, bytealigned: bool | None = None) -> Iterable[int] +.. method:: Bits.findall(bs: BitsType, *, start: int | None = None, end: int | None = None, count: int | None = None, bytealigned: bool | None = None) -> Iterable[int] Searches for all occurrences of *bs* (even overlapping ones) and returns a generator of their bit positions. @@ -148,23 +148,73 @@ Methods [8, 40, 72, 104, 136] -.. classmethod:: Bits.fromstring(s: str, /) -> Bits +.. classmethod:: Bits.from_string(s: str, /) -> Bits Creates a new bitstring from the formatted string *s*. - It is equivalent to creating a new bitstring using *s* as the first parameters, but can be clearer to write and will be slightly faster. + It is equivalent to creating a new bitstring using *s* as the first parameter, but can be clearer to write and will be slightly faster. + The old ``fromstring`` spelling remains available as a compatibility alias. - >>> b1 = Bits('int16=91') - >>> b2 = Bits.fromstring('int16=91') + >>> b1 = Bits('i16=91') + >>> b2 = Bits.from_string('i16=91') >>> b1 == b2 True +.. classmethod:: Bits.from_dtype(dtype: str | Dtype, value: Any, /) -> Bits + + Creates a new bitstring by packing *value* according to *dtype*. :: + + >>> Bits.from_dtype('u10', 85) + Bits('0b0001010101') + +.. classmethod:: Bits.from_bytes(data: bytes | bytearray | memoryview, /, *, length: int | None = None, offset: int = 0) -> Bits + + Creates a new bitstring from a bytes-like object, with optional bit *offset* and *length*. + +.. classmethod:: Bits.from_bools(iterable: Iterable[Any], /) -> Bits + + Creates a new bitstring from an iterable, using ``bool(item)`` for each bit. + +.. classmethod:: Bits.from_zeros(length: int, /) -> Bits + + Creates a new bitstring containing *length* zero bits. + +.. classmethod:: Bits.from_ones(length: int, /) -> Bits + + Creates a new bitstring containing *length* one bits. + +.. classmethod:: Bits.from_joined(sequence: Iterable[BitsType], /) -> Bits + + Creates a new bitstring by concatenating the bitstrings in *sequence*. + +.. classmethod:: Bits.from_file(source: str | Path | BinaryIO, /, *, length: int | None = None, offset: int = 0) -> Bits + + Creates a new bitstring from a file path or binary file object. + +.. classmethod:: Bits.from_tibs(tibs_obj: tibs.Tibs | tibs.Mutibs, /) -> Bits + + Creates a new bitstring from a ``tibs.Tibs`` or ``tibs.Mutibs`` instance. + + This is an interop helper for the lower-level `tibs `_ library that backs bitstring 5. ``Bits`` may share immutable ``tibs.Tibs`` data directly. + Mutable ``tibs.Mutibs`` data is accepted, but is copied before being used by bitstring. + +.. method:: Bits.to_bitarray() -> BitArray + + Returns a mutable copy of the bitstring. + +.. method:: Bits.to_tibs() -> tibs.Tibs + + Returns the data as a ``tibs.Tibs`` instance. + + This is intended for interoperation with the lower-level ``tibs`` library. As ``tibs.Tibs`` is immutable, ``Bits`` can return the underlying object directly. + No ``to_mutibs`` method is provided; use tibs' own conversion methods if you need a mutable tibs object. + .. method:: Bits.join(sequence: Iterable) -> Bits Returns the concatenation of the bitstrings in the iterable *sequence* joined with ``self`` as a separator. :: - >>> s = Bits().join(['0x0001ee', 'uint:24=13', '0b0111']) + >>> s = Bits().join(['0x0001ee', 'u:24=13', '0b0111']) >>> print(s) 0x0001ee00000d7 @@ -190,8 +240,8 @@ Methods ] + trailing_bits = 0x9 - >>> s.pp('int20, hex', width=80, show_offset=False, sep=' / ') - [ + >>> s.pp('i20, hex', width=80, show_offset=False, sep=' / ') + [ -275635 / -107921 / 185209 / 433099 : bcb4d / e5a6f / 2d379 / 69bcb 319066 / 455379 / 497307 / -215842 : 4de5a / 6f2d3 / 7969b / cb4de 370418 / -182378 / -410444 / -137818 : 5a6f2 / d3796 / 9bcb4 / de5a6 @@ -200,7 +250,7 @@ Methods ] - The available formats are any fixed-length dtypes, for example ``'bin'``, ``'oct'``, ``'hex'`` and ``'bytes'`` together with types with explicit lengths such as ``'uint5'`` and ``'float16'``. + The available formats are any fixed-length dtypes, for example ``'bin'``, ``'oct'``, ``'hex'`` and ``'bytes'`` together with types with explicit lengths such as ``'u5'`` and ``'f16'``. A bit length can be specified after the format (with an optional `:`) to give the number of bits represented by each group, otherwise the default is based on the format or formats selected. For the ``'bytes'`` format, characters from the 'Latin Extended-A' unicode block are used for non-ASCII and unprintable characters. @@ -212,11 +262,11 @@ Methods By default the output will have colours added in the terminal. This can be disabled - see :data:`bitstring.options.no_color` for more information. -.. method:: Bits.rfind(bs: BitsType, start: int | None = None, end: int | None = None, bytealigned: bool | None = None) -> Tuple[int] | Tuple[()] +.. method:: Bits.rfind(bs: BitsType, *, start: int | None = None, end: int | None = None, bytealigned: bool | None = None) -> int | None - Searches backwards for *bs* in the current bitstring and sets :attr:`~ConstBitStream.pos` to the start of *bs* and returns it in a tuple if found, otherwise it returns an empty tuple. + Searches backwards for *bs* in the current bitstring and returns the start position if found, otherwise it returns ``None``. - The reason for returning the bit position in a tuple is so that it evaluates as True even if the bit position is zero. This allows constructs such as ``if s.rfind('0xb3'):`` to work as expected. + As bit position zero is a valid result, use ``s.rfind(...) is not None`` when testing whether a match was found. If *bytealigned* is ``True`` then it will look for *bs* only at byte aligned positions. *start* and *end* give the search range and default to ``0`` and :attr:`len` respectively. @@ -224,11 +274,11 @@ Methods >>> s = Bits('0o031544') >>> s.rfind('0b100') - (15,) + 15 >>> s.rfind('0b100', end=17) - (12,) + 12 -.. method:: Bits.split(delimiter: BitsType, start: int | None = None, end: int | None = None, count: int | None = None, bytealigned: bool | None = None) -> Iterable[Bits] +.. method:: Bits.split(delimiter: BitsType, *, start: int | None = None, end: int | None = None, count: int | None = None, bytealigned: bool | None = None) -> Iterable[Bits] Splits the bitstring into sections that start with *delimiter*. Returns a generator for bitstring objects. @@ -240,7 +290,7 @@ Methods >>> [bs.bin for bs in s.split('0x4')] ['', '01000', '01001000', '0100011'] -.. method:: Bits.startswith(bs: BitsType, start: int | None = None, end: int | None = None) -> bool +.. method:: Bits.startswith(bs: BitsType, *, start: int | None = None, end: int | None = None) -> bool Returns ``True`` if the bitstring starts with the sub-string *bs*, otherwise returns ``False``. @@ -250,18 +300,8 @@ Methods >>> s.startswith('0b111011') True -.. method:: Bits.tobitarray() -> bitarray.bitarray - - Returns the bitstring as a ``bitarray`` object. - - Converts the bitstring to an equivalent ``bitarray`` object from the ``bitarray`` package. - This shouldn't be confused with the ``BitArray`` type provided in the ``bitstring`` package - the ``bitarray`` package is a separate third-party way of representing binary objects. - - Note that ``BitStream`` and ``ConstBitStream`` types that have a bit position do support this method but the bit position information will be lost. - - -.. method:: Bits.tobytes() -> bytes +.. method:: Bits.to_bytes() -> bytes Returns the bitstring as a ``bytes`` object. @@ -272,7 +312,7 @@ Methods >>> s = Bits(bytes=b'hello') >>> s += '0b01' - >>> s.tobytes() + >>> s.to_bytes() b'hello@' This is equivalent to casting to a bytes object directly: :: @@ -281,34 +321,34 @@ Methods b'hello@' -.. method:: Bits.tofile(f: BinaryIO) -> None +.. method:: Bits.to_file(f: BinaryIO) -> None Writes the bitstring to the file object *f*, which should have been opened in binary write mode. The data written will be padded at the end with between zero and seven ``0`` bits to make it byte aligned. - The file object remains open so the user must call `.close()` on it once they are finished.:: + The file object remains open so the user must call ``.close()`` on it once they are finished.:: >>> f = open('newfile', 'wb') - >>> Bits('0x1234').tofile(f) + >>> Bits('0x1234').to_file(f) -.. method:: Bits.unpack(fmt: str | list[str | int], **kwargs) -> list[float | int | str | None | Bits] +.. method:: Bits.unpack(fmt: str | list[str | int], **kwargs) -> list[float | int | str | Bits | bool | bytes | None] - Interprets the whole bitstring according to the *fmt* string or iterable and returns a list of bitstring objects. + Interprets the whole bitstring according to the *fmt* string or iterable and returns a list of values. A dictionary or keyword arguments can also be provided. These will replace length identifiers in the format string. *fmt* is an iterable or a string with comma separated tokens that describe how to interpret the next bits in the bitstring. See the :ref:`format_tokens` for details. :: - >>> s = Bits('int4=-1, 0b1110') - >>> i, b = s.unpack('int:4, bin') + >>> s = Bits('i4=-1, 0b1110') + >>> i, b = s.unpack('i:4, bin') If a token doesn't supply a length (as with ``bin`` above) then it will try to consume the rest of the bitstring. Only one such token is allowed. The ``unpack`` method is a natural complement of the :func:`pack` function. :: - s = bitstring.pack('uint10, hex, int13, 0b11', 130, '3d', -23) - a, b, c, d = s.unpack('uint10, hex, int13, bin2') + s = bitstring.pack('u10, hex, i13, 0b11', 130, '3d', -23) + a, b, c, d = s.unpack('u10, hex, i13, bin2') ---- @@ -316,7 +356,7 @@ Properties ---------- The many ways to interpret bitstrings can be accessed via properties. -These properties will be read-only for a ``Bits`` object, but are also writable for derived mutable types such as ``BitArray`` and ``BitStream``. +These properties will be read-only for a ``Bits`` object, but are also writable for derived mutable types such as ``BitArray``. Properties can also have a length in bits appended to them to such as ``u8`` or ``f64`` (for the ``bytes`` property the length is interpreted in bytes instead of bits). These properties with lengths will cause an :exc:`InterpretError` to be raised if the bitstring is not of the specified length. @@ -324,14 +364,14 @@ These properties with lengths will cause an :exc:`InterpretError` to be raised i This list isn't exhaustive - see for example :ref:`Exotic floats` for information on bfloats and many 8-bit and smaller floating point formats. Also see :ref:`exp-golomb` for some interesting variable length integer formats. -Note that the ``bin``, ``oct``, ``hex``, ``int``, ``uint`` and ``float`` properties can all be shortened to their initial letter. +The ``i``, ``u`` and ``f`` properties are the preferred names for bit-wise big-endian integer and floating point interpretations. +The longer ``int``, ``uint`` and ``float`` names remain as compatibility aliases. .. attribute:: Bits.bin :type: str Property for the representation of the bitstring as a binary string. - Can be shortened to just ``b``. .. attribute:: Bits.bool :type: bool @@ -347,7 +387,7 @@ Note that the ``bin``, ``oct``, ``hex``, ``int``, ``uint`` and ``float`` propert When used as a getter the bitstring must be a whole number of byte long or a :exc:`InterpretError` will be raised. - An alternative is to use the :meth:`tobytes` method, which will pad with between zero and seven ``0`` bits to make it byte aligned if needed. :: + An alternative is to use the :meth:`to_bytes` method, which will pad with between zero and seven ``0`` bits to make it byte aligned if needed. :: >>> s = Bits('0x12345678') >>> s.bytes @@ -357,7 +397,6 @@ Note that the ``bin``, ``oct``, ``hex``, ``int``, ``uint`` and ``float`` propert :type: str Property representing the hexadecimal value of the bitstring. - Can be shortened to just ``h``. If the bitstring is not a multiple of four bits long then getting its hex value will raise an :exc:`InterpretError`. :: @@ -365,53 +404,63 @@ Note that the ``bin``, ``oct``, ``hex``, ``int``, ``uint`` and ``float`` propert >>> s.hex 'f0' +.. attribute:: Bits.i + :type: int .. attribute:: Bits.int :type: int + :noindex: Property for the signed two’s complement integer representation of the bitstring. - Can be shortened to just ``i``. + ``int`` is a compatibility alias for ``i``. + The longer endian-specific names ``intbe``, ``intle`` and ``intne`` are + also compatibility aliases for ``ibe``, ``ile`` and ``ine``. -.. attribute:: Bits.intbe +.. attribute:: Bits.ibe :type: int Property for the byte-wise big-endian signed two's complement integer representation of the bitstring. - Only valid for whole-byte bitstrings, in which case it is equal to ``s.int``, otherwise an :exc:`InterpretError` is raised. + Only valid for whole-byte bitstrings, in which case it is equal to ``s.i``, otherwise an :exc:`InterpretError` is raised. -.. attribute:: Bits.intle +.. attribute:: Bits.ile :type: int Property for the byte-wise little-endian signed two's complement integer representation of the bitstring. - Only valid for whole-byte bitstring, in which case it is equal to ``s[::-8].int``, i.e. the integer representation of the byte-reversed bitstring. + Only valid for whole-byte bitstring, in which case it is equal to ``s[::-8].i``, i.e. the integer representation of the byte-reversed bitstring. -.. attribute:: Bits.intne +.. attribute:: Bits.ine :type: int Property for the byte-wise native-endian signed two's complement integer representation of the bitstring. Only valid for whole-byte bitstrings, and will equal either the big-endian or the little-endian integer representation depending on the platform being used. +.. attribute:: Bits.f + :type: float .. attribute:: Bits.float :type: float -.. attribute:: Bits.floatbe + :noindex: +.. attribute:: Bits.fbe :type: float Property for the floating point representation of the bitstring. - Can be shortened to just ``f``. + ``float``, ``floatbe`` and ``fbe`` are compatibility aliases for ``f``. + The longer endian-specific names ``floatle`` and ``floatne`` are also + compatibility aliases for ``fle`` and ``fne``. The bitstring must be 16, 32 or 64 bits long to support the floating point interpretations, otherwise an :exc:`InterpretError` will be raised. If the underlying floating point methods on your machine are not IEEE 754 compliant then using the float interpretations is undefined (this is unlikely unless you're on some very unusual hardware). - The :attr:`float` property is bit-wise big-endian, which as all floats must be whole-byte is exactly equivalent to the byte-wise big-endian :attr:`floatbe`. + The :attr:`f` property is bit-wise big-endian, which as all floats must be whole-byte is exactly equivalent to the byte-wise big-endian :attr:`fbe`. -.. attribute:: Bits.floatle +.. attribute:: Bits.fle :type: float Property for the byte-wise little-endian floating point representation of the bitstring. -.. attribute:: Bits.floatne +.. attribute:: Bits.fne :type: float Property for the byte-wise native-endian floating point representation of the bitstring. @@ -430,7 +479,6 @@ Note that the ``bin``, ``oct``, ``hex``, ``int``, ``uint`` and ``float`` propert :type: str Property for the octal representation of the bitstring. - Can be shortened to just ``o``. If the bitstring is not a multiple of three bits long then getting its octal value will raise a :exc:`InterpretError`. :: @@ -442,23 +490,28 @@ Note that the ``bin``, ``oct``, ``hex``, ``int``, ``uint`` and ``float`` propert '01234567' +.. attribute:: Bits.u + :type: int .. attribute:: Bits.uint :type: int + :noindex: Property for the unsigned base-2 integer representation of the bitstring. - Can be shortened to just ``u``. + ``uint`` is a compatibility alias for ``u``. + The longer endian-specific names ``uintbe``, ``uintle`` and ``uintne`` are + also compatibility aliases for ``ube``, ``ule`` and ``une``. -.. attribute:: Bits.uintbe +.. attribute:: Bits.ube :type: int Property for the byte-wise big-endian unsigned base-2 integer representation of the bitstring. -.. attribute:: Bits.uintle +.. attribute:: Bits.ule :type: int Property for the byte-wise little-endian unsigned base-2 integer representation of the bitstring. -.. attribute:: Bits.uintne +.. attribute:: Bits.une :type: int Property for the byte-wise native-endian unsigned base-2 integer representation of the bitstring. @@ -536,12 +589,12 @@ Special Methods >>> Bits('0o7777') == '0xfff' True - >>> a = Bits(uint=13, length=8) - >>> b = Bits(uint=13, length=10) + >>> a = Bits(u=13, length=8) + >>> b = Bits(u=13, length=10) >>> a == b False - If you have a different criterion you wish to use then code it explicitly, for example ``a.int == b.int`` could be true even if ``a == b`` wasn't (as they could be different lengths). + If you have a different criterion you wish to use then code it explicitly, for example ``a.i == b.i`` could be true even if ``a == b`` wasn't (as they could be different lengths). .. method:: Bits.__getitem__(key) @@ -571,7 +624,7 @@ Special Methods Returns an integer hash of the :class:`Bits`. - This method is not available for the :class:`BitArray` or :class:`BitStream` classes, as only immutable objects should be hashed. You typically won't need to call it directly, instead it is used for dictionary keys and in sets. + This method is not available for the :class:`BitArray` class, as only immutable objects should be hashed. You typically won't need to call it directly, instead it is used for dictionary keys and in sets. .. method:: Bits.__invert__() @@ -581,7 +634,7 @@ Special Methods If the bitstring is empty then an :exc:`Error` will be raised. :: - >>> s = ConstBitStream(‘0b1110010’) + >>> s = Bits('0b1110010') >>> print(~s) 0b0001101 >>> print(~s & s) @@ -689,4 +742,3 @@ Special Methods >>> print(Bits('0x33') ^ '0x0f') 0x3c - diff --git a/doc/bitstream.rst b/doc/bitstream.rst deleted file mode 100644 index 2a3ddd01..00000000 --- a/doc/bitstream.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. currentmodule:: bitstring - -BitStream -========= - -.. class:: BitStream(auto: BitsType | int | None, /, length: int | None = None, offset: int | None = None, pos: int = 0, **kwargs) - - Both the :class:`BitArray` and the :class:`ConstBitStream` classes are base classes for :class:`BitStream` and so all of their methods are also available for :class:`BitStream` objects. The initialiser is the same as for :class:`ConstBitStream`. - - A :class:`BitStream` is a mutable container of bits with methods and properties that allow it to be parsed as a stream of bits. There are no additional methods or properties in this class - see its base classes (:class:`Bits`, :class:`BitArray` and :class:`ConstBitStream`) for details. - - -The ``pos`` will also used as a default for the :meth:`BitArray.overwrite` and :meth:`BitArray.insert` methods. - -The bit position is modified by methods that read bits, as described in :attr:`~ConstBitStream.pos`, but for the mutable ``BitStream`` it is also modified by other methods: - -* If a methods extends the bitstring (``+=``, ``append``) the ``pos`` will move to the end of the bitstring. -* If a method otherwise changes the length of the bitstring (``prepend``, ``insert``, sometimes ``replace``) the ``pos`` becomes invalid and will be reset to ``0``. - diff --git a/doc/conf.py b/doc/conf.py index 80ebc7c0..971fa86b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,7 +12,7 @@ project = 'bitstring' copyright = f'2006 - {year}, Scott Griffiths' author = 'Scott Griffiths' -release = '4.4' +release = '5.0.0_beta1' extensions = [] @@ -31,7 +31,7 @@ html_theme = 'piccolo_theme' html_theme_options = { - # "banner_text": "New requirement of Python 3.8 or later - see release notes for full details.", + "banner_text": "Unreleased bitstring 5.0 docs. The current stable PyPI line is 4.x; pin bitstring<5 for the released API.", "banner_hiding": "permanent", "show_theme_credit": False, "globaltoc_maxdepth": 2, diff --git a/doc/constbitstream.rst b/doc/constbitstream.rst deleted file mode 100644 index 4e141906..00000000 --- a/doc/constbitstream.rst +++ /dev/null @@ -1,205 +0,0 @@ -.. currentmodule:: bitstring - -ConstBitStream -============== - -.. class:: ConstBitStream(auto: BitsType | int | None, /, length: int | None = None, offset: int | None = None, pos: int = 0, **kwargs) - - The :class:`Bits` class is the base class for :class:`ConstBitStream` and so all of its methods are also available for :class:`ConstBitStream` objects. The initialiser is the same as for :class:`Bits` except that an initial bit position :attr:`pos` can be given (defaults to 0). - - A :class:`ConstBitStream` is a :class:`Bits` with added methods and properties that allow it to be parsed as a stream of bits. - ----- - -Reading and parsing ---------------------- - -The :class:`BitStream` and :class:`ConstBitStream` classes contain number of methods for reading the bitstring as if it were a file or stream. Depending on how it was constructed the bitstream might actually be contained in a file rather than stored in memory, but these methods work for either case. - -In order to behave like a file or stream, every bitstream has a property :attr:`~ConstBitStream.pos` which is the current position from which reads occur. :attr:`~ConstBitStream.pos` can range from zero (its default value on construction) to the length of the bitstream, a position from which all reads will fail as it is past the last bit. Note that the :attr:`~ConstBitStream.pos` property isn't considered a part of the bitstream's identity; this allows it to vary for immutable :class:`ConstBitStream` objects and means that it doesn't affect equality or hash values. - -The property :attr:`~ConstBitStream.bytepos` is also available, and is useful if you are only dealing with byte data and don't want to always have to divide the bit position by eight. Note that if you try to use :attr:`~ConstBitStream.bytepos` and the bitstring isn't byte aligned (i.e. :attr:`~ConstBitStream.pos` isn't a multiple of 8) then a :exc:`ByteAlignError` exception will be raised. - -Reading using format strings -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :meth:`~ConstBitStream.read` / :meth:`~ConstBitStream.readlist` methods can also take a format string similar to that used in the auto initialiser. Only one token should be provided to :meth:`~ConstBitStream.read` and a single value is returned. To read multiple tokens use :meth:`~ConstBitStream.readlist`, which unsurprisingly returns a list. - -The format string consists of comma separated tokens that describe how to interpret the next bits in the bitstring. -The tokens are given in :ref:`format_tokens`. - -For example we can read and interpret three quantities from a bitstream with:: - - start_code = s.read('hex32') - width = s.read('uint12') - height = s.read('uint12') - -and we also could have combined the three reads as:: - - start_code, width, height = s.readlist('hex32, 2*uint12') - -where here we are also using a multiplier to combine the format of the second and third tokens. - -You are allowed to use one 'stretchy' token in a :meth:`~ConstBitStream.readlist`. This is a token without a length specified, which will stretch to fill encompass as many bits as possible. This is often useful when you just want to assign something to 'the rest' of the bitstring:: - - a, b, everything_else = s.readlist('intle16, intle24, bits') - -In this example the ``bits`` token will consist of everything left after the first two tokens are read, and could be empty. - -It is an error to use more than one stretchy token, or to use a ``ue``, ``se``, ``uie`` or ``se`` token after a stretchy token (the reason you can't use exponential-Golomb codes after a stretchy token is that the codes can only be read forwards; that is you can't ask "if this code ends here, where did it begin?" as there could be many possible answers). - -The ``pad`` token is a special case in that it just causes bits to be skipped over without anything being returned. This can be useful for example if parts of a binary format are uninteresting:: - - a, b = s.readlist('pad12, uint4, pad4, uint8') - -Peeking -^^^^^^^^ - -In addition to the read methods there are matching peek methods. These are identical to the read except that they do not advance the position in the bitstring to after the read elements. :: - - s = ConstBitStream('0x4732aa34') - if s.peek(8) == '0x47': - t = s.read(16) # t is first 2 bytes '0x4732' - else: - s.find('0x47') - - ----- - -Methods -------- - -.. method:: ConstBitStream.bytealign() -> int - - Aligns to the start of the next byte (so that :attr:`pos` is a multiple of 8) and returns the number of bits skipped. - - If the current position is already byte aligned then it is unchanged. :: - - >>> s = ConstBitStream('0xabcdef') - >>> s.pos += 3 - >>> s.bytealign() - 5 - >>> s.pos - 8 - -.. method:: ConstBitStream.peek(fmt: str | int) -> int | float | str | Bits | bool | bytes | None - - Reads from the current bit position :attr:`pos` in the bitstring according to the *fmt* string or integer and returns the result. - - The bit position is unchanged. - - For information on the format string see the entry for the :meth:`read` method. :: - - >>> s = ConstBitStream('0x123456') - >>> s.peek(16) - ConstBitStream('0x1234') - >>> s.peek('hex8') - '12' - -.. method:: ConstBitStream.peeklist(fmt: str | list[str | int], **kwargs) -> list[int | float | str | Bits | bool | bytes | None] - - Reads from current bit position :attr:`pos` in the bitstring according to the *fmt* string or iterable and returns a list of results. - - A dictionary or keyword arguments can also be provided. These will replace length identifiers in the format string. The position is not advanced to after the read items. - - See the entries for :meth:`read` and :meth:`readlist` for more information. - -.. method:: ConstBitStream.read(fmt: str | int) -> int | float | str | Bits | bool | bytes | None - - Reads from current bit position :attr:`pos` in the bitstring according the format string and returns a single result. If not enough bits are available then a :exc:`ReadError` is raised. - - *fmt* is either a token string that describes how to interpret the next bits in the bitstring or an integer. - If it's an integer then that number of bits will be read, and returned as a new bitstring. - A full list of the tokens is given in :ref:`format_tokens`. - - For example:: - - >>> s = ConstBitStream('0x23ef55302') - >>> s.read('hex12') - '23e' - >>> s.read('bin4') - '1111' - >>> s.read('u5') - 10 - >>> s.read('bits4') - ConstBitStream('0xa') - - The :meth:`~ConstBitStream.read` method is useful for reading exponential-Golomb codes. :: - - >>> s = ConstBitStream('se=-9, ue=4') - >>> s.read('se') - -9 - >>> s.read('ue') - 4 - - The ``pad`` token is not very useful when used in :meth:`~ConstBitStream.read` as it just skips a number of bits and returns ``None``. However when used within :meth:`~ConstBitStream.readlist` or :meth:`~Bits.unpack` it allows unimportant part of the bitstring to be simply ignored. - -.. method:: ConstBitStream.readlist(fmt: str | list[str | int], **kwargs) -> list[int | float | str | Bits | bool | bytes | None] - - Reads from current bit position :attr:`pos` in the bitstring according to the *fmt* string or iterable and returns a list of results. If not enough bits are available then a :exc:`ReadError` is raised. - - A dictionary or keyword arguments can also be provided. These will replace length identifiers in the format string. The position is advanced to after the read items. - - See :ref:`format_tokens` for information on the format strings. - - For multiple items you can separate using commas or given multiple parameters:: - - >>> s = ConstBitStream('0x43fe01ff21') - >>> s.readlist('hex8, uint6') - ['43', 63] - >>> s.readlist(['bin3', 'intle16']) - ['100', -509] - >>> s.pos = 0 - >>> s.readlist('hex:b, uint:d', b=8, d=6) - ['43', 63] - -.. method:: ConstBitStream.readto(bs: BitsType, bytealigned: bool | None = None) -> ConstBitStream - - Reads up to and including the next occurrence of the bitstring *bs* and returns the results. If *bytealigned* is `True` it will look for the bitstring starting only at whole-byte positions. - - Raises a :exc:`ReadError` if *bs* is not found, and :exc:`ValueError` if *bs* is empty. - - >>> s = ConstBitStream('0x47000102034704050647') - >>> s.readto('0x47', bytealigned=True) - ConstBitStream('0x47') - >>> s.readto('0x47', bytealigned=True) - ConstBitStream('0x0001020347') - >>> s.readto('0x47', bytealigned=True) - ConstBitStream('0x04050647') - ----- - -Properties ----------- - -The ``ConstBitStream`` and ``BitStream`` classes have the concept of a current bit position. -This position will be set to zero by default on construction, and will be modified by many of the methods described above as the stream is being read. - -Using :meth:`~Bits.find` or :meth:`~Bits.rfind` will move ``pos`` to the start of the substring if it is found. - -Note that the ``pos`` property isn’t considered a part of the bitstring's identity; this allows it to vary for immutable ``ConstBitStream`` objects and means that it doesn't affect equality or hash values. -It also will be reset to zero if a bitstring is copied. - - -.. attribute:: ConstBitStream.bytepos - :type: int - - Property for setting and getting the current byte position in the bitstring. - The value of ``pos`` will always be ``bytepos * 8`` as the two values are not independent. - - When used as a getter will raise a :exc:`ByteAlignError` if the current position in not byte aligned. - - -.. attribute:: ConstBitStream.pos - :type: int -.. attribute:: ConstBitStream.bitpos - :type: int - - Read and write property for setting and getting the current bit position in the bitstring. Can be set to any value from ``0`` to ``len(s)``. - - The :attr:`pos` and :attr:`bitpos` properties are exactly equivalent - you can use whichever you prefer. :: - - if s.pos < 100: - s.pos += 10 - - diff --git a/doc/dtypes.rst b/doc/dtypes.rst index 789d3f50..b5ff3601 100644 --- a/doc/dtypes.rst +++ b/doc/dtypes.rst @@ -3,14 +3,14 @@ Dtypes ====== -A data type (or 'dtype') concept is used in the bitstring module to encapsulate how to create, parse and present different bit interpretations. +A data type (or 'dtype') concept is used in the bitstring module to encapsulate how to pack, unpack and present different bit interpretations. The properties described above are all examples of dtypes. .. class:: Dtype(token: str | Dtype, /, length: int | None = None, scale: int | float | None = None) Dtypes are immutable and cannot be changed after creation. -The first parameter is a format token string that can optionally include a length. For example ``'ue'``, ``'int'`` or ``'float16'``. +The first parameter is a format token string that can optionally include a length. For example ``'ue'``, ``'i'`` or ``'f16'``. If the first parameter doesn't include a length and one is appropriate, the `length` parameter can be used to specify the length of the dtype. @@ -28,22 +28,22 @@ If you need to use the `scale` parameter then there is no way to specify this in Methods ------- -.. method:: Dtype.build(value: Any, /) -> Bits +.. method:: Dtype.pack(value: Any, /) -> Bits -Create a bitstring from a value. +Pack a value into a bitstring. The *value* parameter should be of a type appropriate to the dtype. >>> d = Dtype('u10') - >>> d.build(85) # Equivalent to: Bits(u10=85) + >>> d.pack(85) # Equivalent to: Bits(u10=85) Bits('0b0001010101') -.. method:: Dtype.parse(b: BitsType, /) -> Any +.. method:: Dtype.unpack(b: BitsType, /) -> Any -Parse a bitstring to find its value. The *b* parameter should be a bitstring of the appropriate length, or an object that can be converted to a bitstring. +Unpack a bitstring to find its value. The *b* parameter should be a bitstring of the appropriate length, or an object that can be converted to a bitstring. >>> d = Dtype('u10') - >>> d.parse('0b0001010101') # Equivalent to: Bits('0b0001010101').u10 + >>> d.unpack('0b0001010101') # Equivalent to: Bits('0b0001010101').u10 85 ---- @@ -93,7 +93,7 @@ A function to read the value of the data type. .. attribute:: Dtype.return_type :type: type -The type of the value returned by the `parse` method, such as ``int``, ``float`` or ``str``. +The type of the value returned by the `unpack` method, such as ``int``, ``float`` or ``str``. .. attribute:: Dtype.scale :type: int | float | None @@ -108,4 +108,4 @@ A function to set the value of the data type. .. attribute:: Dtype.variable_length :type: bool -If True then the length of the data type depends on the data being interpreted, and must not be specified. \ No newline at end of file +If True then the length of the data type depends on the data being interpreted, and must not be specified. diff --git a/doc/exotic_floats.rst b/doc/exotic_floats.rst index 7feb53c2..497a3423 100644 --- a/doc/exotic_floats.rst +++ b/doc/exotic_floats.rst @@ -28,17 +28,17 @@ The more bits in the mantissa, the greater the precision (~significant figures) - 1 + 11 + 52 - IEEE 754 - 10\ :sup:`-308` → 10\ :sup:`308` - - ``'float64'`` / ``'d'`` + - ``'f64'`` / ``'d'`` * - Single precision - 1 + 8 + 23 - IEEE 754 - 10\ :sup:`-38` → 10\ :sup:`38` - - ``'float32'`` / ``'f'`` + - ``'f32'`` / ``'f'`` * - Half precision - 1 + 5 + 10 - IEEE 754 - 6×10\ :sup:`-8` → 65504 - - ``'float16'`` / ``'e'`` + - ``'f16'`` / ``'e'`` * - bfloat - 1 + 8 + 7 - ``-`` @@ -63,7 +63,7 @@ IEEE 8-bit Floating Point Types In bitstring prior to version 4.2 the `p4binary8` and `p3binary8` formats were called `e4m3float` and `e5m2float` respectively. The two formats are almost identical, the difference being the addition of `inf` values that replace the largest positive and negative values that were previously available. - Neither should be confused with the `e4m3mxfp` and `e5m2mxfp` formats from the Open Compute Project described below. + Neither should be confused with the E4M3 and E5M2 MXFP formats from the Open Compute Project described below. The 'binary8' formats are part of an ongoing IEEE standardisation process. @@ -99,7 +99,7 @@ It's remarkable that any useful calculations can be performed, but both inferenc You can easily examine every possible value that these formats can represent using a line like this:: - >>> [Bits(uint=x, length=8).p3binary8 for x in range(256)] + >>> [Bits(u=x, length=8).p3binary8 for x in range(256)] or using the :class:`Array` type it's even more concise - we can create an Array and pretty print all the values with this line:: @@ -178,12 +178,12 @@ There is also a format to use for the scaling factor, an int-like format which i * - E5M2 - 1 + 5 + 2 - 10\ :sup:`-6` → 57344 - - ``'e5m2mxfp'`` + - ``'e5m2mxfp_saturate'`` / ``'e5m2mxfp_overflow'`` * - E4M3 - 1 + 4 + 3 - 2×10\ :sup:`-3` → 448 - - ``'e4m3mxfp'`` + - ``'e4m3mxfp_saturate'`` / ``'e4m3mxfp_overflow'`` * - E3M2 - 1 + 3 + 2 @@ -260,14 +260,12 @@ Values that are out of range after rounding are dealt with as follows: - ``p3binary8`` - Out of range values are set to ``+inf`` or ``-inf``. - ``p4binary8`` - Out of range values are set to ``+inf`` or ``-inf``. -- ``e5m2mxfp`` - Out of range values are dealt with according to the ``bitstring.options.mxfp_overflow`` setting: - - ``'saturate'`` (the default): values are set to the largest positive or negative finite value, as appropriate. Infinities will also be set to the largest finite value, despite the fact that the format has infinities. - - ``'overflow'``: Out of range values are set to ``+inf`` or ``-inf``. -- ``e4m3mxfp`` - Out of range values are dealt with according to the ``bitstring.options.mxfp_overflow`` setting: - - ``'saturate'`` (the default): values are set to the largest positive or negative value, as appropriate. - - ``'overflow'``: Out of range values are set to ``nan``. +- ``e5m2mxfp_saturate`` - Out of range values are set to the largest positive or negative finite value, as appropriate. Infinities will also be set to the largest finite value, despite the fact that the format has infinities. +- ``e5m2mxfp_overflow`` - Out of range values are set to ``+inf`` or ``-inf``. +- ``e4m3mxfp_saturate`` - Out of range values are set to the largest positive or negative value, as appropriate. +- ``e4m3mxfp_overflow`` - Out of range values are set to ``nan``. - ``e3m2mxfp`` - Out of range values are saturated to the largest positive or negative value. - ``e2m3mxfp`` - Out of range values are saturated to the largest positive or negative value. - ``e2m1mxfp`` - Out of range values are saturated to the largest positive or negative value. - ``mxint`` - Out of range values are saturated to the largest positive or negative value. Note that the most negative value in this case is ``-2.0``, which it is optional for an implementation to support. -- ``e8m0mxfp`` - No rounding is done. A ``ValueError`` will be raised if you try to convert a non-representable value. This is because the format is designed as a scaling factor, so it should generally be specified exactly. \ No newline at end of file +- ``e8m0mxfp`` - No rounding is done. A ``ValueError`` will be raised if you try to convert a non-representable value. This is because the format is designed as a scaling factor, so it should generally be specified exactly. diff --git a/doc/exp-golomb.rst b/doc/exp-golomb.rst index c5d26947..f8301b07 100644 --- a/doc/exp-golomb.rst +++ b/doc/exp-golomb.rst @@ -37,7 +37,7 @@ ue The :attr:`~Bits.ue` property interprets the bitstring as a single unsigned exponential-Golomb code and returns an integer. If the bitstring is not exactly one code then an :exc:`InterpretError` is raised instead. If you instead wish to read the next bits in the stream and interpret them as a code use the read function or unpack with a ``ue`` format string. :: - >>> s = BitStream(ue=12) + >>> s = BitArray(ue=12) >>> s.bin '0001101' >>> s.append('ue=3') @@ -49,11 +49,12 @@ se The :attr:`~Bits.se` property does much the same as ``ue`` and the provisos there all apply. The obvious difference is that it interprets the bitstring as a signed exponential-Golomb rather than unsigned. :: - >>> s = BitStream('0x164b') - >>> s.se + >>> bits = Bits('0x164b') + >>> bits.se InterpretError: Bitstring is not a single exponential-Golomb code. - >>> while s.pos < len(s): - ... print(s.read('se')) + >>> r = Reader(bits) + >>> while r.pos < len(r): + ... print(r.read('se')) -5 2 0 @@ -119,4 +120,4 @@ Bit pattern Signed ``...`` ... ============= =========== -I'm sure you can work out the pattern yourself from here! \ No newline at end of file +I'm sure you can work out the pattern yourself from here! diff --git a/doc/functions.rst b/doc/functions.rst index 9ff2d7a3..1a465ef2 100644 --- a/doc/functions.rst +++ b/doc/functions.rst @@ -7,26 +7,26 @@ pack ^^^^ .. function:: pack(format[, *values, **kwargs]) - Packs the values and keyword arguments according to the *format* string and returns a new :class:`BitStream`. + Packs the values and keyword arguments according to the *format* string and returns a new :class:`Bits` object. :param format: string with comma separated tokens - :param values: extra values used to construct the :class:`BitStream` + :param values: extra values used to construct the :class:`Bits` :param kwargs: a dictionary of token replacements - :rtype: BitStream + :rtype: Bits The format string consists of comma separated tokens, see :ref:`format_tokens` and :ref:`compact_format` for details. -The tokens can be 'literals', like ``0xef``, ``0b110``, ``uint8=55``, etc. which just represent a set sequence of bits. +The tokens can be 'literals', like ``0xef``, ``0b110``, ``u8=55``, etc. which just represent a set sequence of bits. They can also have the value missing, in which case the values contained in ``*values`` will be used. :: >>> a = pack('bin3, hex4', '001', 'f') - >>> b = pack('uint10', 33) + >>> b = pack('u10', 33) A dictionary or keyword arguments can also be provided. These will replace items in the format string. :: - >>> c = pack('int:a=b', a=10, b=20) - >>> d = pack('int8=a, bin=b, int4=a', a=7, b='0b110') + >>> c = pack('i:a=b', a=10, b=20) + >>> d = pack('i8=a, bin=b, i4=a', a=7, b='0b110') Plain names can also be used as follows:: @@ -44,16 +44,16 @@ A :exc:`ValueError` will be raised if the ``*values`` are not all used up by the As an example of using just the ``*values`` arguments we can say:: - s = bitstring.pack('hex32, uint12, uint12', '0x000001b3', 352, 288) + s = bitstring.pack('hex32, u12, u12', '0x000001b3', 352, 288) which is equivalent to initialising as:: - s = BitStream('0x0000001b3, uint12=352, uint12=288') + s = Bits('0x000001b3, u12=352, u12=288') The advantage of the pack function is if you want to write more general code for creation. :: def foo(a, b, c, d): - return bitstring.pack('uint8, 0b110, int6, bin, bits', a, b, c, d) + return bitstring.pack('u8, 0b110, i6, bin, bits', a, b, c, d) s1 = foo(12, 5, '0b00000', '') s2 = foo(101, 3, '0b11011', s1) @@ -66,7 +66,7 @@ You can also include keyword, value pairs (or an equivalent dictionary) as the f The values are then packed according to the positions of the keywords in the format string. This is most easily explained with some examples. Firstly the format string needs to contain parameter names:: - format = 'hex32=start_code, uint12=width, uint12=height' + format = 'hex32=start_code, u12=width, u12=height' Then we can make a dictionary with these parameters as keys and pass it to pack:: @@ -78,9 +78,9 @@ Another method is to pass the same information as keywords at the end of pack's s = bitstring.pack(format, width=352, height=288, start_code='0x000001b3') -You can include constant bitstring tokens such as '0x101', '0xff', 'uint7=81' etc. and also use a keyword for the length specifier in the token, for example:: +You can include constant bitstring tokens such as '0x101', '0xff', 'u7=81' etc. and also use a keyword for the length specifier in the token, for example:: - s = bitstring.pack('0xabc, int:n=-1', n=100) + s = bitstring.pack('0xabc, i:n=-1', n=100) Finally it is also possible just to use a keyword as a token:: @@ -92,102 +92,6 @@ Options ------- The bitstring module has an ``options`` object that allows certain module-wide behaviours to be set. -lsb0 -^^^^ - -.. data:: bitstring.options.lsb0 : bool - -By default bit numbering in the bitstring module is done from 'left' to 'right'. That is, from bit ``0`` at the start of the data to bit ``n - 1`` at the end. This allows bitstrings to be treated like an ordinary Python container that is only allowed to contain single bits. - - -The ``lsb0`` option allows bitstrings to use Least Significant Bit Zero -(LSB0) bit numbering; that is the right-most bit in the bitstring will -be bit 0, and the left-most bit will be bit (n-1), rather than the -other way around. LSB0 is a more natural numbering -system in many fields, but is the opposite to Most Significant Bit -Zero (MSB0) numbering which is the natural option when thinking of -bitstrings as standard Python containers. - -For example, if you set a bitstring to be the binary ``010001111`` it will be stored in the same way for MSB0 and LSB0 but slicing, reading, unpacking etc. will all behave differently. - -.. list-table:: MSB0 → - :header-rows: 1 - - * - bit index - - 0 - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - * - value - - ``0`` - - ``1`` - - ``0`` - - ``0`` - - ``0`` - - ``1`` - - ``1`` - - ``1`` - - ``1`` - -In MSB0 everything behaves like an ordinary Python container. Bit zero is the left-most bit and reads/slices happen from left to right. - -.. list-table:: ← LSB0 - :header-rows: 1 - - * - bit index - - 8 - - 7 - - 6 - - 5 - - 4 - - 3 - - 2 - - 1 - - 0 - * - value - - ``0`` - - ``1`` - - ``0`` - - ``0`` - - ``0`` - - ``1`` - - ``1`` - - ``1`` - - ``1`` - -In LSB0 the final, right-most bit is labelled as bit zero. Reads and slices happen from right to left. - -When bitstrings (or slices of bitstrings) are interpreted as integers and other types the left-most bit is considered as the most significant bit. It's important to note that this is the case irrespective of whether the first or last bit is considered the bit zero, so for example if you were to interpret a whole bitstring as an integer, its value would be the same with and without `lsb0` being set to `True`. - -To illustrate this, for the example above this means that the bin and int representations would be ``010001111`` and ``143`` respectively for both MSB0 and LSB0 bit numbering. - -To switch from the default MSB0, use ``bitstring.options.lsb0``. This defaults to ``False`` and unless explicitly stated all examples and documentation related to the bitstring module use the default MSB0 indexing. - - >>> bitstring.options.lsb0 = True - -Slicing is still done with the start bit smaller than the end bit. -For example: - - >>> s = Bits('0b010001111') - >>> s[0:5] # LSB0 so this is the right-most five bits - Bits('0b01111') - >>> s[0] - True - -.. note:: - In some standards and documents using LSB0 notation the slice of the final five bits would be shown as ``s[5:0]``, which is reasonable as bit 5 comes before bit 0 when reading left to right, but this notation isn't used in this module as it clashes too much with the usual Python notation. - -Negative indices work as you'd expect, with the first stored -bit being ``s[-1]`` and the final stored bit being ``s[-n]``. - -Reading, peeking and unpacking of bitstrings are also affected by the ``lsb0`` flag, so reading always increments the bit position, and will move from right to left if ``lsb0`` is ``True``. Because of the way that exponential-Golomb codes are read (with the left-most bits determining the length of the code) these interpretations are not available in LSB0 mode, and using them will raise an exception. - -For ``BitStream`` and ``ConstBitStream`` objects changing the value of ``bitstring.options.lsb0`` invalidates the current position in the bitstring, unless that value is ``0``, and future results are undefined. Basically don't perform reads or change the current bit position before switching the bit numbering system! bytealigned ^^^^^^^^^^^ @@ -198,25 +102,17 @@ A number of methods take a bytealigned parameter to indicate that they should on >>> a = BitArray('0x00 ff 0f ff') >>> a.find('0x0f') - (4,) # found first not on a byte boundary + 4 # found first not on a byte boundary >>> a.find('0x0f', bytealigned=True) - (16,) # forced looking only on byte boundaries + 16 # forced looking only on byte boundaries >>> bitstring.options.bytealigned = True # Change default behaviour >>> a.find('0x0f') - (16,) + 16 >>> a.find('0x0f', bytealigned=False) - (4,) + 4 If you’re only working with bytes then this can help avoid some errors and save some typing. -mxfp_overflow -^^^^^^^^^^^^^ -.. data:: bitstring.options.mxfp_overflow : str - -This option can be used to change the out-of-range behaviour of some 8-bit floating point types. -The default value is ``'saturate'`` but it can also be set to ``'overflow'``. -See :ref:`Exotic floats` for details. - no_color ^^^^^^^^ @@ -228,53 +124,6 @@ See https://no-color.org for more information. The terminal colours can also be turned off by setting ``bitstring.options.no_color`` to ``True``. -using_rust_core -^^^^^^^^^^^^^^^ - -.. data:: bitstring.using_rust_core : bool - -By default the C-based bitarray library is used to optimise the core operations in bitstring. -This being replaced with the `tibs `_ library, which is written in Rust and by the -author of bitstring. -For now both options are available, and bitarray remains the default. - -The ``using_rust_core`` flag is read-only - to try the new library set the ``BITSTRING_USE_RUST_CORE`` environment variable before running your -program, tests or Python interpreter. :: - - % BITSTRING_USE_RUST_CORE=1 python - Python 3.13.5 (main, Jul 23 2025, 00:30:58) [Clang 20.1.4 ] on darwin - Type "help", "copyright", "credits" or "license" for more information. - >>> import bitstring - >>> bitstring.options.using_rust_core - True - -This should work identically in terms of features, except that the ``tobitarray()`` method will be unavailable. - -The plan is to make the Rust core the only option from version 5 of bitstring. - ----- - -Command Line Usage ------------------- - -The bitstring module can be called from the command line to perform simple operations. For example:: - - $ python -m bitstring int16=-400 - 0xfe70 - - $ python -m bitstring float32=0.2 bin - 00111110010011001100110011001101 - - $ python -m bitstring 0xff "3*0b01,0b11" uint - 65367 - - $ python -m bitstring hex=01, uint12=352.hex - 01160 - -Command-line parameters are concatenated and a bitstring created from them. If the final parameter is either an interpretation string or ends with a ``.`` followed by an interpretation string then that interpretation of the bitstring will be used when printing it. If no interpretation is given then the bitstring is just printed. - ----- - Exceptions ---------- @@ -297,4 +146,3 @@ Exceptions .. exception:: ReadError(Error, IndexError) Reading or peeking past the end of a bitstring. - diff --git a/doc/index.rst b/doc/index.rst index 205c6cda..aa192856 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -9,6 +9,12 @@ It has been maintained since 2006. +.. important:: + + These docs describe the upcoming bitstring 5.0 work. + The current released PyPI line is still bitstring 4.x. - see the 'stable' version + of these docs. + .. image:: https://github.com/scott-griffiths/bitstring/actions/workflows/.github/workflows/ci.yml/badge.svg :target: https://github.com/scott-griffiths/bitstring/actions/workflows/ci.yml @@ -54,9 +60,11 @@ The :ref:`quick_reference` provides a summary of the classes and their methods. The :ref:`reference` section has a complete list of all the classes, methods, properties and functions of the bitstring module, together with short examples for many items. +If you are moving code from bitstring 4.x to 5.x, see :ref:`upgrading_to_version_5`. + * :class:`Bits` - an immutable container of bits. * :class:`BitArray` - adds mutating methods to ``Bits``. - * :class:`BitStream` - adds a bit position and read methods. + * :class:`Reader` - wraps ``Bits`` or ``BitArray`` with a bit position and read methods. * :class:`Array` - an array of bitstrings of the same type. @@ -78,29 +86,29 @@ Mixed format bitstrings If you have binary data (or want to construct it) from multiple types then you could use the :class:`BitArray` class. The example below constructs a 28 bit bitstring from a hexadecimal string, then unpacks it into multiple bit interpretations. -It also demonstrates how it can be flexibly modified and sliced using standard notation, and how properties such as ``bin`` and ``float`` can be used to interpret the data. +It also demonstrates how it can be flexibly modified and sliced using standard notation, and how properties such as ``bin`` and ``f`` can be used to interpret the data. :: >>> s = bitstring.BitArray('0x4f8e220') - >>> s.unpack('uint12, hex8, bin') + >>> s.unpack('u12, hex8, bin') [1272, 'e2', '00100000'] >>> '0b11000' in s True >>> s += 'f32=0.001' >>> s.bin '010011111000111000100010000000111010100000110001001001101111' - >>> s[-32:].float + >>> s[-32:].f 0.0010000000474974513 -The module also supplies the :class:`BitStream` class, which adds a bit position so that objects can also be read from, searched in, and navigated in, similar to a file or stream. +The module also supplies the :class:`Reader` class, which wraps a bitstring with a bit position so that it can be read from, searched in, and navigated in, similar to a file or stream. Bitstrings are designed to be as lightweight as possible and can be considered to be just a list of binary digits. They are however stored efficiently - although there are a variety of ways of creating and viewing the binary data, the bitstring itself just stores the byte data, and all views are calculated as needed, and are not stored as part of the object. -The different views or interpretations on the data are accessed through properties such as :attr:`~Bits.hex`, :attr:`~Bits.bin` and :attr:`~Bits.int`, and an extensive set of functions is supplied for modifying, navigating and analysing the binary data. +The different views or interpretations on the data are accessed through properties such as :attr:`~Bits.hex`, :attr:`~Bits.bin` and :attr:`~Bits.i`, and an extensive set of functions is supplied for modifying, navigating and analysing the binary data. -There are also companion classes called :class:`Bits` and :class:`ConstBitStream` which are immutable versions of :class:`BitArray` and :class:`BitStream` respectively. +:class:`Bits` is immutable and :class:`BitArray` is mutable. A :class:`Reader` can wrap either class, so the reading position is kept separate from the bit data itself. See the reference documentation for full details. Arrays of bitstrings @@ -115,22 +123,22 @@ You can also easily change the data's interpretation, convert to another format, :: - >>> a = bitstring.Array('uint16', [0, 1, 4, 6, 11, 2, 8, 7]) + >>> a = bitstring.Array('u16', [0, 1, 4, 6, 11, 2, 8, 7]) >>> a.data BitArray('0x0000000100040006000b000200080007') - >>> b = a.astype('uint5') + >>> b = a.astype('u5') >>> b.data BitArray('0x0048658907') - >>> a.tolist() == b.tolist() + >>> a.to_list() == b.to_list() True You can also take and set slices as you'd expect, and apply operations to each element in the ``Array``. :: >>> a[::2] *= 5 >>> a - Array('uint16', [0, 1, 20, 6, 55, 2, 40, 7]) + Array('u16', [0, 1, 20, 6, 55, 2, 40, 7]) >>> a >> 2 - Array('uint16', [0, 0, 5, 1, 13, 0, 10, 1]) + Array('u16', [0, 0, 5, 1, 13, 0, 10, 1]) Installation and download @@ -153,5 +161,3 @@ Credits Created by Scott Griffiths in 2006 to help with ad hoc parsing and creation of compressed video files. Maintained and expanded ever since as it became unexpectedly popular. Thanks to all those who have contributed ideas, code and bug reports over the years. These docs are styled using the `Piccolo theme `_. - - diff --git a/doc/interpretation.rst b/doc/interpretation.rst index 07d2bb74..ce72cf81 100644 --- a/doc/interpretation.rst +++ b/doc/interpretation.rst @@ -21,9 +21,9 @@ If you're in an interactive session then the pretty-print method :meth:`~Bits.pp Properties ---------- -Many of the more commonly used interpretations have single letter equivalents. -The ``hex``, ``bin``, ``oct``, ``int``, ``uint`` and ``float`` properties can be shortened to ``h``, ``b``, ``o``, ``i``, ``u`` and ``f`` respectively. -Properties can have bit lengths appended to them to make properties such as ``f64``, ``u32`` or ``floatle32``. +The ``i``, ``u`` and ``f`` properties are the preferred names for bit-wise big-endian integer and floating point interpretations. +The longer ``int``, ``uint`` and ``float`` names remain as compatibility aliases. +Properties can have bit lengths appended to them to make properties such as ``f64``, ``u32``, ``bin12`` or ``fle32``. When used as a getter these just add an extra check on the bitstring's length - if the bitstring is not the stated length then an :exc:`InterpretError` is raised. When used as a setter they define the new length of the bitstring. :: @@ -44,7 +44,7 @@ The most fundamental interpretation is perhaps as a binary string (a ‘bitstrin >>> a.bin '000100100011' - >>> b.b + >>> b.bin '111' Note that the initial zeros are significant; for bitstrings the zeros are just as important as the ones! @@ -55,7 +55,7 @@ If the bitstring does not have a length that is a multiple of four bits then an >>> a.hex '123' - >>> b.h + >>> b.hex ValueError: Cannot convert to hex unambiguously - not multiple of 4 bits. For an octal interpretation use the :attr:`~Bits.oct` property. @@ -64,7 +64,7 @@ If the bitstring does not have a length that is a multiple of three then an :exc >>> a.oct '0443' - >>> b.o + >>> b.oct '7' >>> (b + '0b0').oct ValueError: Cannot convert to octal unambiguously - not multiple of 3 bits. @@ -72,61 +72,61 @@ If the bitstring does not have a length that is a multiple of three then an :exc Integer types ------------- -To interpret the bitstring as a binary (base-2) bit-wise big-endian unsigned integer (i.e. a non-negative integer) use the :attr:`~Bits.uint` property. +To interpret the bitstring as a binary (base-2) bit-wise big-endian unsigned integer (i.e. a non-negative integer) use the :attr:`~Bits.u` property. - >>> a.uint + >>> a.u 283 >>> b.u 7 -For byte-wise big-endian, little-endian and native-endian interpretations use :attr:`~Bits.uintbe`, :attr:`~Bits.uintle` and :attr:`~Bits.uintne` respectively. These will raise a :exc:`ValueError` if the bitstring is not a whole number of bytes long. :: +For byte-wise big-endian, little-endian and native-endian interpretations use :attr:`~Bits.ube`, :attr:`~Bits.ule` and :attr:`~Bits.une` respectively. These will raise a :exc:`ValueError` if the bitstring is not a whole number of bytes long. :: >>> s = BitArray('0x000001') - >>> s.uint # bit-wise big-endian + >>> s.u # bit-wise big-endian 1 - >>> s.uintbe # byte-wise big-endian + >>> s.ube # byte-wise big-endian 1 - >>> s.uintle # byte-wise little-endian + >>> s.ule # byte-wise little-endian 65536 - >>> s.uintne # byte-wise native-endian (will be 1 on a big-endian platform!) + >>> s.une # byte-wise native-endian (will be 1 on a big-endian platform!) 65536 -For a two's complement interpretation as a base-2 signed integer use the :attr:`~Bits.int` property. If the first bit of the bitstring is zero then the :attr:`~Bits.int` and :attr:`~Bits.uint` interpretations will be equal, otherwise the :attr:`~Bits.int` will represent a negative number. :: +For a two's complement interpretation as a base-2 signed integer use the :attr:`~Bits.i` property. If the first bit of the bitstring is zero then the :attr:`~Bits.i` and :attr:`~Bits.u` interpretations will be equal, otherwise the :attr:`~Bits.i` will represent a negative number. :: - >>> a.int + >>> a.i 283 >>> b.i -1 -For byte-wise big, little and native endian signed integer interpretations use :attr:`~Bits.intbe`, :attr:`~Bits.intle` and :attr:`~Bits.intne` respectively. These work in the same manner as their unsigned counterparts described above. +For byte-wise big, little and native endian signed integer interpretations use :attr:`~Bits.ibe`, :attr:`~Bits.ile` and :attr:`~Bits.ine` respectively. These work in the same manner as their unsigned counterparts described above. bytes ----- A common need is to retrieve the raw bytes from a bitstring for further processing or for writing to a file. For this use the :py:attr:`~Bits.bytes` interpretation, which returns a ``bytes`` object. -If the length of the bitstring isn't a multiple of eight then a :exc:`ValueError` will be raised. This is because there isn't an unequivocal representation as ``bytes``. You may prefer to use the method :meth:`~Bits.tobytes` as this will be pad with between one and seven zero bits up to a byte boundary if necessary. :: +If the length of the bitstring isn't a multiple of eight then a :exc:`ValueError` will be raised. This is because there isn't an unequivocal representation as ``bytes``. You may prefer to use the method :meth:`~Bits.to_bytes` as this will pad with between one and seven zero bits up to a byte boundary if necessary. :: - >>> open('somefile', 'wb').write(a.tobytes()) + >>> open('somefile', 'wb').write(a.to_bytes()) >>> open('anotherfile', 'wb').write(('0x0'+a).bytes) - >>> a1 = BitArray(filename='somefile') + >>> a1 = BitArray.from_file('somefile') >>> a1.hex '1230' - >>> a2 = BitArray(filename='anotherfile') + >>> a2 = BitArray.from_file('anotherfile') >>> a2.hex '0123' -Note that the :meth:`~Bits.tobytes` method automatically padded with four zero bits at the end, whereas for the other example we explicitly padded at the start to byte align before using the :attr:`~Bits.bytes` property. +Note that the :meth:`~Bits.to_bytes` method automatically padded with four zero bits at the end, whereas for the other example we explicitly padded at the start to byte align before using the :attr:`~Bits.bytes` property. Floating point types -------------------- -For a floating point interpretation use the :attr:`~Bits.float` property. This uses the IEEE 754 floating point representation and will only work if the bitstring is 16, 32 or 64 bits long. +For a floating point interpretation use the :attr:`~Bits.f` property. This uses the IEEE 754 floating point representation and will only work if the bitstring is 16, 32 or 64 bits long. -Different endiannesses are provided via :attr:`~Bits.floatle` and :attr:`~Bits.floatne`. -Note that as floating point interpretations are only valid on whole-byte bitstrings there is no difference between the bit-wise big-endian :attr:`~Bits.float` and the byte-wise big-endian :attr:`~Bits.floatbe`. +Different endiannesses are provided via :attr:`~Bits.fle` and :attr:`~Bits.fne`. +Note that as floating point interpretations are only valid on whole-byte bitstrings there is no difference between the bit-wise big-endian :attr:`~Bits.f` and the byte-wise big-endian :attr:`~Bits.fbe`. Note also that standard floating point numbers in Python are stored in 64 bits, so use this size if you wish to avoid rounding errors. @@ -135,7 +135,7 @@ Other floating point types -------------------------- A range of floating point types that are mostly used in machine learning are also availabe. -They include ``bfloat16`` which is a truncated ``float32``, together with IEEE 8-bit formats and a range of OCP Microscaling 8-bit, 6-bit and 4-bit formats. +They include ``bfloat16`` which is a truncated ``f32``, together with IEEE 8-bit formats and a range of OCP Microscaling 8-bit, 6-bit and 4-bit formats. See :ref:`Exotic floats` for more information. @@ -147,4 +147,3 @@ Some variable length integer types are supported. The lengths of these types depends upon the data being read and they are mainly used in video codecs. See :ref:`exp-golomb` for more information. - diff --git a/doc/introduction.rst b/doc/introduction.rst index c7c4c5cc..adfa0d06 100644 --- a/doc/introduction.rst +++ b/doc/introduction.rst @@ -10,28 +10,28 @@ Introduction The bitstring classes --------------------- -Five classes are provided by the bitstring module, four are simple containers of bits: +Four classes are provided by the bitstring module. Two are simple containers of bits: * :class:`Bits`: This is the most basic class. It is immutable and so its contents can't be changed after creation. * :class:`BitArray`: This adds mutating methods to its base class. -* :class:`ConstBitStream`: This adds methods and properties to allow the bits to be treated as a stream of bits, with a bit position and reading/parsing methods. -* :class:`BitStream`: This is the most versatile class, having both the bitstream methods and the mutating methods. :class:`Bits` and :class:`BitArray` are intended to loosely mirror the ``bytes`` and ``bytearray`` types in Python. -The term 'bitstring' is used in this documentation to refer generically to any of these four classes. +The term 'bitstring' is used in this documentation to refer generically to either of these classes. -The fifth class is :class:`Array` which is a container of fixed-length bitstrings. +The :class:`Reader` class wraps either a :class:`Bits` or :class:`BitArray` object with a bit position for sequential reading. +The :class:`Array` class is a container of fixed-length bitstrings. The rest of this introduction mostly concerns the more basic types - for more details on :class:`Array` you can go directly to the reference documentation, but understanding how bit format strings are specified will be helpful. To summarise when to use each class: -* If you need to change the contents of the bitstring then you must use :class:`BitArray` or :class:`BitStream`. Truncating, replacing, inserting, appending etc. are not available for the const classes. -* If you need to use a bitstring as the key in a dictionary or as a member of a ``set`` then you must use :class:`Bits` or a :class:`ConstBitStream`. As :class:`BitArray` and :class:`BitStream` objects are mutable they do not support hashing and so cannot be used in these ways. -* If you are creating directly from a file then a :class:`BitArray` or :class:`BitStream` will read the whole file into memory whereas a :class:`Bits` or :class:`ConstBitStream` will not, so using the const classes allows extremely large files to be examined. +* If you need to change the contents of the bitstring then use :class:`BitArray`. Truncating, replacing, inserting, appending etc. are not available for :class:`Bits`. +* If you need to use a bitstring as the key in a dictionary or as a member of a ``set`` then use :class:`Bits`. :class:`BitArray` objects are mutable and so cannot be used in these ways. +* If you are creating directly from a file then :class:`BitArray` will read the whole file into memory whereas :class:`Bits` will not, so using :class:`Bits` allows extremely large files to be examined. +* If you need sequential reads then create a :class:`Reader` around either kind of bitstring. * If you don't need the extra functionality of a particular class then the simpler ones might be faster and more memory efficient. The fastest and most memory efficient class is :class:`Bits`. -The :class:`Bits` class is the base class of the other three class. This means that ``isinstance(s, Bits)`` will be true if ``s`` is an instance of any of the four classes. +The :class:`Bits` class is the base class of :class:`BitArray`. This means that ``isinstance(s, Bits)`` will be true for both bit container classes. ---- @@ -41,30 +41,29 @@ Constructing bitstrings When initialising a bitstring you need to specify at most one initialiser. This can either be the first parameter in the constructor ('auto' initialisation, described below), or using a keyword argument for a data type. -``Bits(auto, /, length: Optional[int], offset: Optional[int], **kwargs)`` +``Bits(auto, /, length: int | None = None, offset: int | None = None, **kwargs)`` Some of the keyword arguments that can be used are: * ``bytes`` : A ``bytes`` object, for example read from a binary file. * ``hex``, ``oct``, ``bin``: Hexadecimal, octal or binary strings. -* ``int``, ``uint``: Signed or unsigned bit-wise big-endian binary integers. -* ``intle``, ``uintle``: Signed or unsigned byte-wise little-endian binary integers. -* ``intbe``, ``uintbe``: Signed or unsigned byte-wise big-endian binary integers. -* ``intne``, ``uintne``: Signed or unsigned byte-wise native-endian binary integers. -* ``float`` / ``floatbe``, ``floatle``, ``floatne``: Big, little and native endian floating point numbers. +* ``i``, ``u``: Signed or unsigned bit-wise big-endian binary integers. ``int`` and ``uint`` are compatibility aliases. +* ``ile``, ``ule``: Signed or unsigned byte-wise little-endian binary integers. +* ``ibe``, ``ube``: Signed or unsigned byte-wise big-endian binary integers. +* ``ine``, ``une``: Signed or unsigned byte-wise native-endian binary integers. +* ``f`` / ``fbe``, ``fle``, ``fne``: Big, little and native endian floating point numbers. ``float`` is a compatibility alias. * ``bool`` : A boolean (i.e. True or False). -* ``filename`` : Directly from a file, without reading into memory if using :class:`Bits` or :class:`ConstBitStream`. There are also various other flavours of 16-bit, 8-bit and smaller floating point types (see :ref:`Exotic floats`) and exponential-Golomb integer types (see :ref:`exp-golomb`). -The ``hex``, ``oct``, ``bin``, ``float``, ``int`` and ``uint`` can all be shortened to just their initial letters. +The ``f``, ``i`` and ``u`` names are preferred for bit-wise big-endian floats and integers. The data type name can be combined with its length if appropriate, or the length can be specified separately. For example:: a = Bits(hex='deadbeef') - b = BitArray(f32=100.25) # or = BitArray(float=100.25, length=32) - c = ConstBitStream(filename='a_big_file') + b = BitArray(f32=100.25) # or = BitArray(f=100.25, length=32) + c = Bits.from_file('a_big_file') d = Bits(u12=105) e = BitArray(bool=True) @@ -80,26 +79,28 @@ See the entry on :func:`pack` for more information. The auto initialiser -------------------- -The first parameter when creating a bitstring is a positional only parameter, referred to as 'auto', that can be a variety of types: +The first parameter when creating a bitstring is a positional only parameter, referred to as 'auto'. +It is most commonly a formatted string, a bytes-like object, or another bitstring. +For other common construction modes use explicit factory methods: -* An iterable, whose elements will be evaluated as booleans (imagine calling ``bool()`` on each item) and the bits set to ``1`` for ``True`` items and ``0`` for ``False`` items. -* A positive integer, used to create a bitstring of that many zero bits. -* A file object, opened in binary mode, from which the bitstring will be formed. -* A ``bytearray`` or ``bytes`` object. -* An ``array`` object from the built-in ``array`` module. This is used after being converted to it's constituent byte data via its ``tobytes`` method. -* A ``bitarray`` or ``frozenbitarray`` object from the 3rd party ``bitarray`` package. +* :meth:`Bits.from_bools` for an iterable whose elements are evaluated as booleans. +* :meth:`Bits.from_zeros` or :meth:`Bits.from_ones` for repeated zero or one bits. +* :meth:`Bits.from_file` for a file path or binary file object. +* :meth:`Bits.from_bytes` for bytes-like data with optional bit offset and length. +* :meth:`Bits.from_joined` for concatenating many bitstrings. +* :meth:`Bits.from_tibs` for interoperation with lower-level ``tibs.Tibs`` or ``tibs.Mutibs`` objects. If it is a string then that string will be parsed into tokens to construct the binary data: * Starting with ``'0x'`` or ``'hex='`` implies hexadecimal. e.g. ``'0x013ff'``, ``'hex=013ff'`` * Starting with ``'0o'`` or ``'oct='`` implies octal. e.g. ``'0o755'``, ``'oct=755'`` * Starting with ``'0b'`` or ``'bin='`` implies binary. e.g. ``'0b0011010'``, ``'bin=0011010'`` -* Starting with ``'int'`` or ``'uint'`` followed by a length in bits and ``'='`` gives base-2 integers. e.g. ``'uint8=255'``, ``'int4=-7'`` -* To get big, little and native-endian whole-byte integers append ``'be'``, ``'le'`` or ``'ne'`` respectively to the ``'uint'`` or ``'int'`` identifier. e.g. ``'uintle32=1'``, ``'intne16=-23'`` -* For floating point numbers use ``'float'`` followed by the length in bits and ``'='`` and the number. The default is big-endian, but you can also append ``'be'``, ``'le'`` or ``'ne'`` as with integers. e.g. ``'float64=0.2'``, ``'floatle32=-0.3e12'`` +* Starting with ``'i'`` or ``'u'`` followed by a length in bits and ``'='`` gives base-2 integers. e.g. ``'u8=255'``, ``'i4=-7'`` +* To get big, little and native-endian whole-byte integers append ``'be'``, ``'le'`` or ``'ne'`` respectively to the ``'u'`` or ``'i'`` identifier. e.g. ``'ule32=1'``, ``'ine16=-23'`` +* For floating point numbers use ``'f'`` followed by the length in bits and ``'='`` and the number. The default is big-endian, but you can also append ``'be'``, ``'le'`` or ``'ne'`` as with integers. e.g. ``'f64=0.2'``, ``'fle32=-0.3e12'`` * Starting with ``'ue='``, ``'uie='``, ``'se='`` or ``'sie='`` implies an exponential-Golomb coded integer. e.g. ``'ue=12'``, ``'sie=-4'`` -Multiples tokens can be joined by separating them with commas, so for example ``'uint4=4, 0b1, se=-1'`` represents the concatenation of three elements. +Multiples tokens can be joined by separating them with commas, so for example ``'u4=4, 0b1, se=-1'`` represents the concatenation of three elements. Parentheses and multiplicative factors can also be used, for example ``'2*(0b10, 0xf)'`` is equivalent to ``'0b10, 0xf, 0b10, 0xf'``. The multiplying factor must come before the thing it is being used to repeat. @@ -128,20 +129,21 @@ Methods that need another bitstring as a parameter will also 'auto' promote, for which illustrates a variety of methods promoting strings, iterables and a bytes object to bitstrings. +``tibs.Tibs`` and ``tibs.Mutibs`` objects are also promoted; mutable ``tibs.Mutibs`` values are copied before bitstring uses them. -Anything that can be used as the first parameter of the ``Bits`` constructor can be automatically promoted to a bitstring where one is expected, with the exception of integers. -Integers won't be promoted, but instead will raise a ``TypeError``:: +Promotion is intentionally broader than the public constructor, but integers are still not promoted. +They raise a ``TypeError`` because an integer is more likely to be a mistaken value than a deliberate request for that many zero bits:: - >>> a = BitArray(100) # Create bitstring with 100 zeroed bits. + >>> a = BitArray.from_zeros(100) # Create bitstring with 100 zeroed bits. >>> a += 0xff # TypeError - 0xff is the same as the integer 255. >>> a += '0xff' # Probably what was meant - append eight '1' bits. - >>> a += Bits(255) # If you really want to do it then code it explicitly. + >>> a += Bits.from_zeros(255) # If you really want to do it then code it explicitly. ``BitsType`` ^^^^^^^^^^^^ -.. class:: BitsType(Bits | str | Iterable[Any] | bool | BinaryIO | bytearray | bytes | memoryview | bitarray.bitarray) +.. class:: BitsType(Bits | str | tibs.Tibs | tibs.Mutibs | Iterable[Any] | BinaryIO | bytearray | bytes | memoryview) The ``BitsType`` type is used in the documentation in a number of places where an object of any type that can be promoted to a bitstring is acceptable. @@ -199,41 +201,41 @@ As with ``hex`` and ``bin``, the 'auto' initialiser will work if the octal strin From an integer ^^^^^^^^^^^^^^^ - >>> e = BitArray(uint=45, length=12) - >>> f = BitArray(int=-1, length=7) + >>> e = BitArray(u=45, length=12) + >>> f = BitArray(i=-1, length=7) >>> e.bin '000000101101' >>> f.bin '1111111' -For initialisation with signed and unsigned binary integers (``int`` and ``uint`` respectively) the ``length`` parameter is mandatory, and must be large enough to contain the integer. -So for example if ``length`` is 8 then ``uint`` can be in the range 0 to 255, while ``int`` can range from -128 to 127. +For initialisation with signed and unsigned binary integers (``i`` and ``u`` respectively) the ``length`` parameter is mandatory, and must be large enough to contain the integer. +So for example if ``length`` is 8 then ``u`` can be in the range 0 to 255, while ``i`` can range from -128 to 127. Two's complement is used to represent negative numbers. -The 'auto' initialiser can be used by giving the length in bits immediately after the ``int`` or ``uint`` token, followed by an equals sign then the value:: +The 'auto' initialiser can be used by giving the length in bits immediately after the ``i`` or ``u`` token, followed by an equals sign then the value:: - >>> e = BitArray('uint12=45') - >>> f = BitArray('int7=-1') + >>> e = BitArray('u12=45') + >>> f = BitArray('i7=-1') -The ``uint`` and ``int`` names can be shortened to just ``u`` and ``i`` respectively. For mutable bitstrings you can change value by assigning to a property with a length:: +For mutable bitstrings you can change value by assigning to a property with a length:: >>> e = BitArray() >>> e.u12 = 45 >>> f = BitArray() >>> f.i7 = -1 -The plain ``int`` and ``uint`` initialisers are bit-wise big-endian. That is to say that the most significant bit comes first and the least significant bit comes last, so the unsigned number one will have a ``1`` as its final bit with all other bits set to ``0``. These can be any number of bits long. For whole-byte bitstring objects there are more options available with different endiannesses. +The plain ``i`` and ``u`` initialisers are bit-wise big-endian. That is to say that the most significant bit comes first and the least significant bit comes last, so the unsigned number one will have a ``1`` as its final bit with all other bits set to ``0``. These can be any number of bits long. For whole-byte bitstring objects there are more options available with different endiannesses. Big and little-endian integers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - >>> big_endian = BitArray(uintbe=1, length=16) - >>> little_endian = BitArray(uintle=1, length=16) - >>> native_endian = BitArray(uintne=1, length=16) + >>> big_endian = BitArray(ube=1, length=16) + >>> little_endian = BitArray(ule=1, length=16) + >>> native_endian = BitArray(une=1, length=16) There are unsigned and signed versions of three additional 'endian' types. The unsigned versions are used above to create three bitstrings. -The first of these, ``big_endian``, is equivalent to just using the plain bit-wise big-endian ``uint`` initialiser, except that all ``intbe`` or ``uintbe`` interpretations must be of whole-byte bitstrings, otherwise a :exc:`ValueError` is raised. +The first of these, ``big_endian``, is equivalent to just using the plain bit-wise big-endian ``u`` initialiser, except that all ``ibe`` or ``ube`` interpretations must be of whole-byte bitstrings, otherwise a :exc:`ValueError` is raised. The second, ``little_endian``, is interpreted as least significant byte first, i.e. it is a byte reversal of ``big_endian``. So we have:: @@ -247,19 +249,19 @@ Finally we have ``native_endian``, which will equal either ``big_endian`` or ``l From a floating point number ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - >>> f1 = BitArray(float=10.3, length=32) - >>> f2 = BitArray('float64=5.4e31') + >>> f1 = BitArray(f=10.3, length=32) + >>> f2 = BitArray('f64=5.4e31') Floating point numbers can be used for initialisation provided that the bitstring is 16, 32 or 64 bits long. Standard Python floating point numbers are 64 bits long, so if you use 32 bits then some accuracy could be lost. The 16 bit version has very limited range and is used mainly in specialised areas such as machine learning. The exact bits used to represent the floating point number will conform to the IEEE 754 standard, even if the machine being used does not use that standard internally. -Similar to the situation with integers there are big and little endian versions. The plain ``float`` is big endian and so ``floatbe`` is just an alias. +Similar to the situation with integers there are big and little endian versions. The plain ``f`` is big endian, with ``float`` and ``fbe`` available as aliases. As with other initialisers you can also 'auto' initialise, as demonstrated with the second example below:: - >>> little_endian = BitArray(floatle=0.0, length=64) - >>> native_endian = BitArray('floatne:32=-6.3') + >>> little_endian = BitArray(fle=0.0, length=64) + >>> native_endian = BitArray('fne:32=-6.3') See also :ref:`Exotic floats` for information on other floating point representations that are supported (bfloat and different 8-bit and smaller float formats). @@ -287,14 +289,14 @@ You may wonder why you would bother doing this in this case as the syntax is sli From raw byte data ^^^^^^^^^^^^^^^^^^ -Using the length and offset parameters to specify the length in bits and an offset at the start to be ignored is particularly useful when initialising from raw data or from a file. :: +Using the length and offset parameters to specify the length in bits and an offset at the start to be ignored is particularly useful when creating bitstrings from raw data. :: - a = BitArray(bytes=b'\x00\x01\x02\xff', length=28, offset=1) - b = BitArray(bytes=open("somefile", 'rb').read()) + a = BitArray.from_bytes(b'\x00\x01\x02\xff', length=28, offset=1) + b = BitArray.from_bytes(bytearray([0, 1, 2, 255])) The ``length`` parameter is optional; it defaults to the length of the data in bits (and so will be a multiple of 8). You can use it to truncate some bits from the end of the bitstring. The ``offset`` parameter is also optional and is used to truncate bits at the start of the data. -You can also use a ``bytearray`` or a ``bytes`` object, either explicitly with a ``bytes=some_bytearray`` keyword or via the 'auto' initialiser:: +You can also use a ``bytearray`` or a ``bytes`` object, either via :meth:`Bits.from_bytes` or via the 'auto' initialiser:: c = BitArray(a_bytearray_object) d = BitArray(b'\x23g$5') @@ -303,24 +305,24 @@ You can also use a ``bytearray`` or a ``bytes`` object, either explicitly with a From a file ^^^^^^^^^^^ -Using the ``filename`` initialiser allows a file to be analysed without the need to read it all into memory. The way to create a file-based bitstring is:: +Using :meth:`Bits.from_file` with a path allows a file to be analysed without the need to read it all into memory. The way to create a file-based bitstring is:: - p = Bits(filename="my200GBfile") + p = Bits.from_file("my200GBfile") This will open the file in binary read-only mode. The file will only be read as and when other operations require it, and the contents of the file will not be changed by any operations. If only a portion of the file is needed then the ``offset`` and ``length`` parameters (specified in bits) can be used. Note that we created a :class:`Bits` here rather than a :class:`BitArray`, as they have quite different behaviour in this case. The immutable :class:`Bits` will never read the file into memory (except as needed by other operations), whereas if we had created a :class:`BitArray` then the whole of the file would immediately have been read into memory. This is because in creating a :class:`BitArray` you are implicitly saying that you want to modify it, and so it needs to be in memory. -It's also possible to use the 'auto' initialiser for file objects. It's as simple as:: +It's also possible to use :meth:`Bits.from_file` with a file object. It's as simple as:: - f = open('my200GBfile', 'rb') - p = Bits(f) + with open('my200GBfile', 'rb') as f: + p = Bits.from_file(f) .. note:: - For the immutable types ``Bits`` and ``ConstBitstream`` the file is memory mapped (mmap) in a read-only mode for efficiency. + For immutable ``Bits`` the file is memory mapped (mmap) in a read-only mode for efficiency. This behaves slightly differently depending on the platform; in particular Windows will lock the file against any further writing whereas Unix-like systems will not. - This means that you won't be able to write to the file from Windows OS while the ``Bits`` or ``ConstBitStream`` object exists. + This means that you won't be able to write to the file from Windows OS while the ``Bits`` object exists. - The work-arounds for this are to either (i) Delete the object before opening the file for writing, (ii) Use either ``BitArray`` or ``BitStream`` which will read the whole file into memory or (iii) Stop using Windows (or run in WSL). + The work-arounds for this are to either (i) Delete the object before opening the file for writing, (ii) Use ``BitArray`` which will read the whole file into memory or (iii) Stop using Windows (or run in WSL). diff --git a/doc/optimisation.rst b/doc/optimisation.rst index 1939607b..f648131a 100644 --- a/doc/optimisation.rst +++ b/doc/optimisation.rst @@ -3,38 +3,36 @@ Optimisation Techniques ======================= -The :mod:`bitstring` module aims to be as fast as reasonably possible, and since version 4.1 has used the ``bitarray`` C extension to power its core. +The :mod:`bitstring` module aims to be as fast as reasonably possible, and since version 5.0 has used the ``tibs`` Rust module to power its core. There are however some pointers you should follow to make your code efficient, so if you need things to run faster then this is the section for you. Use combined read and interpretation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When parsing a bitstring one way to write code is in the following style:: +When parsing with a :class:`Reader`, one way to write code is in the following style:: - width = s.read(12).uint - height = s.read(12).uint - flags = s.read(4).bin + width = r.read(12).u + height = r.read(12).u + flags = r.read(4).bin -This works fine, but is not very quick. The problem is that the call to :meth:`~ConstBitStream.read` constructs and returns a new bitstring, which then has to be interpreted. The new bitstring isn't used for anything else and so creating it is wasted effort. Instead it is better to use a string parameter that does the read and interpretation together:: +This works fine, but is not very quick. The problem is that the call to :meth:`~Reader.read` constructs and returns a new bitstring, which then has to be interpreted. The new bitstring isn't used for anything else and so creating it is wasted effort. Instead it is better to use a string parameter that does the read and interpretation together:: - width = s.read('uint12') - height = s.read('uint12') - flags = s.read('bin4') + width = r.read('u12') + height = r.read('u12') + flags = r.read('bin4') This is much faster, although probably not as fast as the combined call:: - width, height, flags = s.readlist('uint12, uint12, bin4') + width, height, flags = r.read_list('u12, u12, bin4') Choose the simplest class you can ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you don't need to modify your bitstring after creation then prefer the immutable :class:`Bits` over the mutable :class:`BitArray`. This is typically the case when parsing, or when creating directly from files. +If you don't need to modify your bitstring after creation then prefer the immutable :class:`Bits` over the mutable :class:`BitArray`. This is typically the case when parsing, or when creating directly from files. Wrap it in a :class:`Reader` if you need sequential reads. The speed difference between the classes is noticeable, and there are also memory usage optimisations that are made if objects are known to be immutable. -You should also prefer :class:`ConstBitStream` to :class:`BitStream` if you won't need to modify any bits. - One anti-pattern to watch out for is using ``+=`` on a :class:`Bits` object. For example, don't do this:: s = Bits() @@ -53,13 +51,13 @@ Use dedicated functions for bit setting and checking If you need to set or check individual bits then there are special functions for this. For example one way to set bits would be:: - s = BitArray(1000) + s = BitArray.from_zeros(1000) for p in [14, 34, 501]: s[p] = '0b1' This creates a 1000 bit bitstring and sets three of the bits to '1'. Unfortunately the crucial line spends most of its time creating a new bitstring from the '0b1' string. You could make it slightly quicker by using ``s[p] = True``, but it is much faster (and I mean at least an order of magnitude) to use the :meth:`~BitArray.set` method:: - s = BitArray(1000) + s = BitArray.from_zeros(1000) s.set(True, [14, 34, 501]) As well as :meth:`~BitArray.set` and :meth:`~BitArray.invert` there are also checking methods :meth:`~Bits.all` and :meth:`~Bits.any`. So rather than using :: @@ -80,4 +78,4 @@ If the pattern of setting or getting can be expressed as a ``range`` then it is you should just write :: - s.set(True, range(0, len(s), 2)) \ No newline at end of file + s.set(True, range(0, len(s), 2)) diff --git a/doc/quick_reference.rst b/doc/quick_reference.rst index d83520cc..90673ffe 100644 --- a/doc/quick_reference.rst +++ b/doc/quick_reference.rst @@ -8,39 +8,29 @@ Quick Reference This section gives a summary of the bitstring module's classes, functions and attributes. -There are four main classes that are bit containers, so that each element is a single bit. -They differ based on whether they can be modified after creation and on whether they have the concept of a current bit position. +There are two main classes that are bit containers, so that each element is a single bit. +They differ based on whether they can be modified after creation. Sequential reading is provided by wrapping either class in a :class:`Reader`. .. |nbsp| unicode:: 0xa0 :trim: .. list-table:: - :widths: 20 15 15 50 + :widths: 20 15 50 :header-rows: 1 * - Class - Mutable? - - Streaming methods? - * - :ref:`bits_quick_reference` - - |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| ✘ - |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| ✘ - An efficient, immutable container of bits. * - :ref:`bitarray_quick_reference` - |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| ✔ - - |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| ✘ - Like ``Bits`` but it can be changed after creation. - * - :ref:`constbitstream_quick_reference` - - |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| ✘ - - |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| ✔ - - Immutable like ``Bits`` but with a bit position and reading methods. - * - :ref:`bitstream_quick_reference` - - |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| ✔ - - |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| |nbsp| ✔ - - Mutable like ``BitArray`` but with a bit position and reading methods. -The final class is a flexible container whose elements are fixed-length bitstrings. +The :class:`Reader` class wraps either bit container with a current bit position for reading. +The :class:`Array` class is a flexible container whose elements are fixed-length bitstrings. .. list-table:: :widths: 20 15 15 50 @@ -61,18 +51,18 @@ Bits :class:`Bits` is the most basic class and is just a container of bits. It is immutable, so once created its value cannot change. -``Bits(auto, /, length: Optional[int], offset: Optional[int], **kwargs)`` +``Bits(auto, /, length: int | None = None, offset: int | None = None, **kwargs)`` -The first parameter (usually referred to as `auto`) can be many different types, including parsable strings, a file handle, a bytes or bytearray object, an integer or an iterable. +The first parameter (usually referred to as `auto`) is most often a parsable string, a bytes-like object, or another bitstring. For zero-filled, bool-iterable, joined or file-based construction prefer the explicit factory methods below. -A single initialiser from `kwargs` can be used instead of ``auto``, including ``bin``, ``hex``, ``oct``, ``bool``, ``uint``, ``int``, ``float``, ``bytes`` and ``filename``. +A single initialiser from `kwargs` can be used instead of ``auto``, including ``bin``, ``hex``, ``oct``, ``bool``, ``u``, ``i``, ``f`` and ``bytes``. Examples:: Bits('0xef') - Bits(float=-50.5, length=32) - Bits('uint10=99') - Bits(uint=99, length=10) + Bits(f=-50.5, length=32) + Bits('u10=99') + Bits(u=99, length=10) Methods ^^^^^^^ @@ -85,15 +75,24 @@ Methods * :meth:`~Bits.endswith` -- Return whether the bitstring ends with a sub-bitstring. * :meth:`~Bits.find` -- Find a sub-bitstring in the current bitstring. * :meth:`~Bits.findall` -- Find all occurrences of a sub-bitstring in the current bitstring. -* :meth:`~Bits.fromstring` -- Create a bitstring from a formatted string. +* :meth:`~Bits.from_bools` -- Create a bitstring from an iterable of bool-like values. +* :meth:`~Bits.from_bytes` -- Create a bitstring from bytes-like data. +* :meth:`~Bits.from_dtype` -- Create a bitstring by packing a value according to a dtype. +* :meth:`~Bits.from_file` -- Create a bitstring from a file path or binary file object. +* :meth:`~Bits.from_joined` -- Create a bitstring by concatenating a sequence. +* :meth:`~Bits.from_ones` -- Create a bitstring of one bits. +* :meth:`~Bits.from_string` -- Create a bitstring from a formatted string. +* :meth:`~Bits.from_tibs` -- Create a bitstring from a ``tibs.Tibs`` or ``tibs.Mutibs`` instance. +* :meth:`~Bits.from_zeros` -- Create a bitstring of zero bits. * :meth:`~Bits.join` -- Join bitstrings together using current bitstring. * :meth:`~Bits.pp` -- Pretty print the bitstring. * :meth:`~Bits.rfind` -- Seek backwards to find a sub-bitstring. * :meth:`~Bits.split` -- Create generator of chunks split by a delimiter. * :meth:`~Bits.startswith` -- Return whether the bitstring starts with a sub-bitstring. -* :meth:`~Bits.tobitarray` -- Return bitstring as a ``bitarray`` object from the `bitarray `_ package. -* :meth:`~Bits.tobytes` -- Return bitstring as bytes, padding if needed. -* :meth:`~Bits.tofile` -- Write bitstring to file, padding if needed. +* :meth:`~Bits.to_bitarray` -- Return a mutable copy. +* :meth:`~Bits.to_bytes` -- Return bitstring as bytes, padding if needed. +* :meth:`~Bits.to_file` -- Write bitstring to file, padding if needed. +* :meth:`~Bits.to_tibs` -- Return the data as a ``tibs.Tibs`` instance. * :meth:`~Bits.unpack` -- Interpret bits using format string. @@ -119,23 +118,23 @@ Properties These read-only properties of the ``Bits`` object are interpretations of the binary data and are calculated as required. Many require the bitstring to be specific lengths. -* :attr:`~Bits.bin` / ``b`` -- The bitstring as a binary string. +* :attr:`~Bits.bin` -- The bitstring as a binary string. * :attr:`~Bits.bool` -- For single bit bitstrings, interpret as True or False. * :attr:`~Bits.bytes` -- The bitstring as a bytes object. -* :attr:`~Bits.float` / ``floatbe`` / ``f`` -- Interpret as a big-endian floating point number. -* :attr:`~Bits.floatle` -- Interpret as a little-endian floating point number. -* :attr:`~Bits.floatne` -- Interpret as a native-endian floating point number. -* :attr:`~Bits.hex` / ``h`` -- The bitstring as a hexadecimal string. -* :attr:`~Bits.int` / ``i`` -- Interpret as a two's complement signed integer. -* :attr:`~Bits.intbe` -- Interpret as a big-endian signed integer. -* :attr:`~Bits.intle` -- Interpret as a little-endian signed integer. -* :attr:`~Bits.intne` -- Interpret as a native-endian signed integer. +* :attr:`~Bits.f` / ``float`` / ``fbe`` -- Interpret as a big-endian floating point number. +* :attr:`~Bits.fle` -- Interpret as a little-endian floating point number. +* :attr:`~Bits.fne` -- Interpret as a native-endian floating point number. +* :attr:`~Bits.hex` -- The bitstring as a hexadecimal string. +* :attr:`~Bits.i` / ``int`` -- Interpret as a two's complement signed integer. +* :attr:`~Bits.ibe` -- Interpret as a big-endian signed integer. +* :attr:`~Bits.ile` -- Interpret as a little-endian signed integer. +* :attr:`~Bits.ine` -- Interpret as a native-endian signed integer. * :attr:`~Bits.len` -- Length of the bitstring in bits. -* :attr:`~Bits.oct` / ``o`` -- The bitstring as an octal string. -* :attr:`~Bits.uint` / ``u`` -- Interpret as a two's complement unsigned integer. -* :attr:`~Bits.uintbe` -- Interpret as a big-endian unsigned integer. -* :attr:`~Bits.uintle` -- Interpret as a little-endian unsigned integer. -* :attr:`~Bits.uintne` -- Interpret as a native-endian unsigned integer. +* :attr:`~Bits.oct` -- The bitstring as an octal string. +* :attr:`~Bits.u` / ``uint`` -- Interpret as a two's complement unsigned integer. +* :attr:`~Bits.ube` -- Interpret as a big-endian unsigned integer. +* :attr:`~Bits.ule` -- Interpret as a little-endian unsigned integer. +* :attr:`~Bits.une` -- Interpret as a native-endian unsigned integer. There are also various other flavours of 16-bit, 8-bit and smaller floating point types (see :ref:`Exotic floats`) and exponential-Golomb integer types (see :ref:`exp-golomb`) that are not listed here for brevity. @@ -168,6 +167,7 @@ All of the methods listed above for the ``Bits`` class are available, plus: * :meth:`~BitArray.rol` -- Rotate bits to the left. * :meth:`~BitArray.ror` -- Rotate bits to the right. * :meth:`~BitArray.set` -- Set bit(s) to 1 or 0. +* :meth:`~BitArray.to_bits` -- Return an immutable copy. Additional special methods ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -189,55 +189,35 @@ The special methods available for the ``Bits`` class are all available, plus som ---- -.. _constbitstream_quick_reference: +.. _reader_quick_reference: -ConstBitStream --------------- - -``Bits`` ⟶ ``ConstBitStream`` - -:class:`ConstBitStream` adds a bit position and methods to read and navigate in an immutable bitstream. -If you wish to use streaming methods on a large file without changing it then this is often the best class to use. - -The constructor is the same as for ``Bits`` / ``BitArray`` but with an optional current bit position. - -``ConstBitStream(auto, length: Optional[int], offset: Optional[int], pos: int = 0, **kwargs)`` - -All of the methods, special methods and properties listed above for the ``Bits`` class are available, plus: - -Additional methods -^^^^^^^^^^^^^^^^^^ - -* :meth:`~ConstBitStream.bytealign` -- Align to next byte boundary. -* :meth:`~ConstBitStream.peek` -- Peek at and interpret next bits as a single item. -* :meth:`~ConstBitStream.peeklist` -- Peek at and interpret next bits as a list of items. -* :meth:`~ConstBitStream.read` -- Read and interpret next bits as a single item. -* :meth:`~ConstBitStream.readlist` -- Read and interpret next bits as a list of items. -* :meth:`~ConstBitStream.readto` -- Read up to and including next occurrence of a bitstring. - -Additional properties -^^^^^^^^^^^^^^^^^^^^^ - -* :attr:`~ConstBitStream.bytepos` -- The current byte position in the bitstring. -* :attr:`~ConstBitStream.pos` -- The current bit position in the bitstring. - ----- - -.. _bitstream_quick_reference: +Reader +------ +:class:`Reader` wraps a :class:`Bits` or :class:`BitArray` with a current bit position. -BitStream ---------- +``Reader(bits: Bits | BitArray, pos: int = 0)`` -``Bits`` ⟶ ``BitArray / ConstBitStream`` ⟶ ``BitStream`` +Methods +^^^^^^^ +* :meth:`~Reader.byte_align` -- Align to next byte boundary. +* :meth:`~Reader.find` -- Find a sub-bitstring and move ``pos`` if found. +* :meth:`~Reader.peek` -- Peek at and interpret next bits as a single item. +* :meth:`~Reader.peek_list` -- Peek at and interpret next bits as a list of items. +* :meth:`~Reader.read` -- Read and interpret next bits as a single item. +* :meth:`~Reader.read_list` -- Read and interpret next bits as a list of items. +* :meth:`~Reader.read_to` -- Read up to and including next occurrence of a bitstring. +* :meth:`~Reader.rfind` -- Search backwards and move ``pos`` if found. -:class:`BitStream` contains all of the 'stream' elements of ``ConstBitStream`` and adds all of the mutating methods of ``BitArray``. -The constructor is the same as for ``ConstBitStream``. -It has all the methods, special methods and properties of the ``Bits``, ``BitArray`` and ``ConstBitArray`` classes. +Properties +^^^^^^^^^^ -It is the most general of the four classes, but it is usually best to choose the simplest class for your use case. +* :attr:`~Reader.bits` -- The wrapped ``Bits`` or ``BitArray`` object. +* :attr:`~Reader.bitpos` -- Alias for :attr:`~Reader.pos`. +* :attr:`~Reader.bytepos` -- The current byte position. +* :attr:`~Reader.pos` -- The current bit position. ---- @@ -252,9 +232,9 @@ It is similar to the ``array`` type in the `array H', [1, 10, 20]) - Array('float16', a_file_object) - Array('int4', stored_bytes) + Array('f16', a_file_object) + Array('i4', stored_bytes) Methods @@ -277,14 +257,14 @@ Methods * :meth:`~Array.count` -- Count the number of occurrences of a value. * :meth:`~Array.equals` -- Compare with another Array for exact equality. * :meth:`~Array.extend` -- Append multiple items to the end of the Array from an iterable. -* :meth:`~Array.fromfile` -- Append items read from a file object. +* :meth:`~Array.from_file` -- Append items read from a file object. * :meth:`~Array.insert` -- Insert an item at a given position. * :meth:`~Array.pop` -- Return and remove an item. * :meth:`~Array.pp` -- Pretty print the Array. * :meth:`~Array.reverse` -- Reverse the order of all items. -* :meth:`~Array.tobytes` -- Return Array data as bytes object, padding with zero bits at the end if needed. -* :meth:`~Array.tofile` -- Write Array data to a file, padding with zero bits at the end if needed. -* :meth:`~Array.tolist` -- Return Array items as a list. +* :meth:`~Array.to_bytes` -- Return Array data as bytes object, padding with zero bits at the end if needed. +* :meth:`~Array.to_file` -- Write Array data to a file, padding with zero bits at the end if needed. +* :meth:`~Array.to_list` -- Return Array items as a list. Special methods ^^^^^^^^^^^^^^^ @@ -330,24 +310,24 @@ Mutating versions of many of the methods are also available. * :meth:`[] ` -- Set an element or slice. * :meth:`del ` -- Delete an element or slice. -* :meth:`+= ` -- Add value to each element in-place. -* :meth:`-= ` -- Subtract value from each element in-place. -* :meth:`*= ` -- Multiply each element by a value in-place. -* :meth:`/= ` -- Divide each element by a value in-place. -* :meth:`//= ` -- Floor divide each element by a value in-place. -* :meth:`%= ` -- Take modulus of each element with a value in-place. -* :meth:`\<\<= ` -- Shift bits of each element to the left in-place. -* :meth:`>>= ` -- Shift bits of each element to the right in-place. -* :meth:`&= ` -- In-place bit-wise AND of each element. -* :meth:`|= ` -- In-place bit-wise OR of each element. -* :meth:`^= ` -- In-place bit-wise XOR of each element. +* :meth:`+= ` -- Add value to each element in-place. +* :meth:`-= ` -- Subtract value from each element in-place. +* :meth:`*= ` -- Multiply each element by a value in-place. +* :meth:`/= ` -- Divide each element by a value in-place. +* :meth:`//= ` -- Floor divide each element by a value in-place. +* :meth:`%= ` -- Take modulus of each element with a value in-place. +* :meth:`\<\<= ` -- Shift bits of each element to the left in-place. +* :meth:`>>= ` -- Shift bits of each element to the right in-place. +* :meth:`&= ` -- In-place bit-wise AND of each element. +* :meth:`|= ` -- In-place bit-wise OR of each element. +* :meth:`^= ` -- In-place bit-wise XOR of each element. Example:: - >>> a = Array('float16', [1.5, 2.5, 7, 1000]) - >>> a[::2] *= 3.0 # Multiply every other float16 value in-place + >>> a = Array('f16', [1.5, 2.5, 7, 1000]) + >>> a[::2] *= 3.0 # Multiply every other f16 value in-place >>> a - Array('float16', [4.5, 2.5, 21.0, 1000.0]) + Array('f16', [4.5, 2.5, 21.0, 1000.0]) The bit-wise logical operations (``&``, ``|``, ``^``) are performed on each element with a ``Bits`` object, which must have the same length as the ``Array`` elements. @@ -361,7 +341,7 @@ Properties * :attr:`~Array.data` -- The complete binary data in a ``BitArray`` object. Can be freely modified. * :attr:`~Array.dtype` -- The data type or typecode. Can be freely modified. * :attr:`~Array.itemsize` -- The length *in bits* of a single item. Read only. -* :attr:`~Array.trailing_bits` -- If the data length is not a multiple of the `dtype` length, this ``BitArray`` gives the leftovers at the end of the data. +* :attr:`~Array.trailing_bits` -- If the data length is not a multiple of :attr:`~Array.itemsize`, this ``BitArray`` gives the leftovers at the end of the data. ---- @@ -370,9 +350,9 @@ Properties Dtype ----- -A data type (or 'dtype') concept is used in the bitstring module to encapsulate how to create, parse and present different bit interpretations. +A data type (or 'dtype') concept is used in the bitstring module to encapsulate how to pack, unpack and present different bit interpretations. -``Dtype(token: str, /, length: int | None, scale: int | float | None = None)`` +``Dtype(token: str, /, length: int | None = None, scale: int | float | None = None)`` Creates a :class:`Dtype` object. Dtypes are immutable and cannot be changed after creation. @@ -388,8 +368,8 @@ The `scale` parameter can be used to specify a multiplicative scaling factor for Methods ^^^^^^^ -* :meth:`~Dtype.build` -- Create a bitstring from a value. -* :meth:`~Dtype.parse` -- Parse a bitstring to find its value. +* :meth:`~Dtype.pack` -- Pack a value into a bitstring. +* :meth:`~Dtype.unpack` -- Unpack a bitstring to find its value. Properties @@ -404,7 +384,7 @@ All properties are read-only. * :attr:`~Dtype.length` -- The length of the data type in units of `bits_per_item`. * :attr:`~Dtype.name` -- A string giving the name of the data type. * :attr:`~Dtype.read_fn` -- A function to read the value of the data type. -* :attr:`~Dtype.return_type` -- The type of the value returned by the `parse` method. +* :attr:`~Dtype.return_type` -- The type of the value returned by the `unpack` method. * :attr:`~Dtype.scale` -- The multiplicative scale applied when interpreting the data. * :attr:`~Dtype.set_fn` -- A function to set the value of the data type. * :attr:`~Dtype.variable_length` -- If True then the length of the data type varies, and shouldn't be specified. @@ -422,31 +402,55 @@ Format strings are used when constructing bitstrings, as well as reading, packin They can also be auto promoted to bitstring when appropriate - see :ref:`auto_init`. -=================== =============================================================================== -``'int:n'`` ``n`` bits as a signed integer. -``'uint:n'`` ``n`` bits as an unsigned integer. -``'intbe:n'`` ``n`` bits as a byte-wise big-endian signed integer. -``'uintbe:n'`` ``n`` bits as a byte-wise big-endian unsigned integer. -``'intle:n'`` ``n`` bits as a byte-wise little-endian signed integer. -``'uintle:n'`` ``n`` bits as a byte-wise little-endian unsigned integer. -``'intne:n'`` ``n`` bits as a byte-wise native-endian signed integer. -``'uintne:n'`` ``n`` bits as a byte-wise native-endian unsigned integer. -``'float:n'`` ``n`` bits as a big-endian floating point number (same as ``floatbe``). -``'floatbe:n'`` ``n`` bits as a big-endian floating point number (same as ``float``). -``'floatle:n'`` ``n`` bits as a little-endian floating point number. -``'floatne:n'`` ``n`` bits as a native-endian floating point number. -``'hex:n'`` ``n`` bits as a hexadecimal string. -``'oct:n'`` ``n`` bits as an octal string. -``'bin:n'`` ``n`` bits as a binary string. -``'bits:n'`` ``n`` bits as a new bitstring. -``'bytes:n'`` ``n`` bytes as a ``bytes`` object. -``'bool[:1]'`` next bit as a boolean (True or False). -``'pad:n'`` next ``n`` bits will be ignored (padding). Only applicable when reading, not creating. -=================== =============================================================================== +.. list-table:: + :widths: 24 76 + + * - ``'i:n'`` + - ``n`` bits as a signed integer. + * - ``'u:n'`` + - ``n`` bits as an unsigned integer. + * - ``'ibe:n'`` + - ``n`` bits as a byte-wise big-endian signed integer. + * - ``'ube:n'`` + - ``n`` bits as a byte-wise big-endian unsigned integer. + * - ``'ile:n'`` + - ``n`` bits as a byte-wise little-endian signed integer. + * - ``'ule:n'`` + - ``n`` bits as a byte-wise little-endian unsigned integer. + * - ``'ine:n'`` + - ``n`` bits as a byte-wise native-endian signed integer. + * - ``'une:n'`` + - ``n`` bits as a byte-wise native-endian unsigned integer. + * - ``'f:n'`` + - ``n`` bits as a big-endian floating point number. + * - ``'float:n'`` + - Alias for ``'f:n'``. + * - ``'fbe:n'`` + - Alias for ``'f:n'``. + * - ``'fle:n'`` + - ``n`` bits as a little-endian floating point number. + * - ``'fne:n'`` + - ``n`` bits as a native-endian floating point number. + * - ``'hex:n'`` + - ``n`` bits as a hexadecimal string. + * - ``'oct:n'`` + - ``n`` bits as an octal string. + * - ``'bin:n'`` + - ``n`` bits as a binary string. + * - ``'bits:n'`` + - ``n`` bits as a new bitstring. + * - ``'bytes:n'`` + - ``n`` bytes as a ``bytes`` object. + * - ``'bool[:1]'`` + - Next bit as a boolean (True or False). + * - ``'pad:n'`` + - Next ``n`` bits will be ignored (padding). Only applicable when reading, not creating. The ``':'`` before the length is optional, and is mostly omitted in the documentation, except where it improves readability. -The ``hex``, ``bin``, ``oct``, ``int``, ``uint`` and ``float`` properties can all be shortened to just their initial letter. +The longer ``int``, ``uint`` and ``float`` names remain as compatibility aliases for ``i``, ``u`` and ``f``. +Longer endian-specific names such as ``intle``, ``uintbe`` and ``floatne`` also remain as compatibility aliases for ``ile``, ``ube`` and ``fne``. +The ``fbe`` and ``floatbe`` names are aliases for ``f``. See also :ref:`Exotic floats` and :ref:`exp-golomb` for other types that can be used in format token strings. @@ -456,7 +460,7 @@ Bitstring literals To make a literal quantity (one that directly represents a sequence of bits) you can use any of the format tokens above followed by an ``'='`` and a value to initialise with. For example:: - s = BitArray('float32=10.125, int7=-9') + s = BitArray('f32=10.125, i7=-9') s.append('hex:abc') You can also create binary, octal and hexadecimal literals by starting a string with ``'0b'``, ``'0o'`` and ``'0x'`` respectively:: @@ -504,9 +508,9 @@ This is followed by at least one of these format characters: The exact type is determined by combining the endianness character with the format character, but rather than give an exhaustive list a single example should explain: ======== ====================================== =========== -``'>h'`` Big-endian 16 bit signed integer ``intbe16`` -``'h'`` Big-endian 16 bit signed integer ``ibe16`` +``'h'`` on big-endian systems, and equal the little-endian ``'>> r = Reader(Bits('0x160120f')) + >>> r.read(12).hex + '160' + >>> r.pos = 0 + >>> r.read('u12') + 352 + >>> r.read_list('u12, bin3') + [288, '111'] + +If a read fails then :attr:`pos` is restored to its value before the read. + +The position is deliberately lax: assigning to :attr:`pos`, :attr:`bitpos` or +:attr:`bytepos` stores the integer value without checking it against the wrapped +bitstring length. Reading methods will raise an error if the position is not +usable when they need it. + +Methods +------- + +.. method:: Reader.read(fmt: str | int | Dtype) -> int | float | str | Bits | bool | bytes | None + + Reads from :attr:`pos` according to *fmt* and advances the position. + + If *fmt* is an integer then that many bits are returned as a bitstring. If it + is a string or :class:`Dtype` then the corresponding dtype is read. + +.. method:: Reader.read_list(fmt: str | list[str | int | Dtype], **kwargs) -> list[int | float | str | Bits | bool | bytes | None] + + Reads one or more format tokens and returns a list of values. + +.. method:: Reader.peek(fmt: str | int | Dtype) -> int | float | str | Bits | bool | bytes | None + + Like :meth:`read`, but leaves :attr:`pos` unchanged. + +.. method:: Reader.peek_list(fmt: str | list[str | int | Dtype], **kwargs) -> list[int | float | str | Bits | bool | bytes | None] + + Like :meth:`read_list`, but leaves :attr:`pos` unchanged. + +.. method:: Reader.read_to(bs: BitsType, *, bytealigned: bool | None = None) -> Bits + + Reads up to and including the next occurrence of *bs*. + +.. method:: Reader.byte_align() -> int + + Aligns :attr:`pos` to the next byte boundary and returns the number of bits + skipped. + +.. method:: Reader.find(bs: BitsType, *, start: int | None = None, end: int | None = None, bytealigned: bool | None = None) -> int | None + + Searches the wrapped bitstring and sets :attr:`pos` to the match position if + *bs* is found. Returns the match position, or ``None`` if not found. + +.. method:: Reader.rfind(bs: BitsType, *, start: int | None = None, end: int | None = None, bytealigned: bool | None = None) -> int | None + + Searches backwards and sets :attr:`pos` to the match position if *bs* is + found. Returns the match position, or ``None`` if not found. + +Properties +---------- + +.. attribute:: Reader.bits + :type: Bits | BitArray + + The wrapped bitstring object. This is the original object, not a copy. + +.. attribute:: Reader.pos + :type: int +.. attribute:: Reader.bitpos + :type: int + + The current bit position. These aliases store any integer value. + +.. attribute:: Reader.bytepos + :type: int + + The current byte position. Reading this property requires :attr:`pos` to be + byte aligned and raises :exc:`ByteAlignError` otherwise. diff --git a/doc/reference.rst b/doc/reference.rst index 8a83d79d..a923c45a 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -18,9 +18,7 @@ This section gives more information and usually examples. dtypes bits bitarray - constbitstream - bitstream + reader array functions - diff --git a/doc/upgrading_to_version_5.rst b/doc/upgrading_to_version_5.rst new file mode 100644 index 00000000..92c8c942 --- /dev/null +++ b/doc/upgrading_to_version_5.rst @@ -0,0 +1,435 @@ +.. currentmodule:: bitstring + +.. _upgrading_to_version_5: + +###################### +Upgrading to version 5 +###################### + +This guide is for code being moved from bitstring 4.4.x to bitstring 5.x. +It focuses on changes that may need source updates. + +The main change in version 5 is that bitstrings no longer store a stream +position. The bit data is represented by :class:`Bits` or :class:`BitArray`, +and sequential reading is handled by a separate :class:`Reader`. + +Minimum Python version +====================== + +bitstring 5 requires Python 3.10 or later. If your package metadata pins +bitstring and Python versions, update both together, for example:: + + requires-python = ">=3.10" + dependencies = ["bitstring>=5"] + +The ``bitarray`` dependency has also been removed. The core bit storage is now +provided by the required ``tibs`` dependency. + +Replace stream classes with Reader +================================== + +The ``ConstBitStream`` and ``BitStream`` classes have been removed. + +For immutable data, wrap a :class:`Bits` object in a :class:`Reader`:: + + # bitstring 4 + s = ConstBitStream("0x160120f") + value = s.read("uint12") + + # bitstring 5 + r = Reader(Bits("0x160120f")) + value = r.read("u12") + +For mutable data, wrap a :class:`BitArray`. The wrapped object is available as +:attr:`Reader.bits`, and it is the original object rather than a copy:: + + # bitstring 4 + s = BitStream("0x001122") + first = s.read("uint8") + s.append("0xff") + + # bitstring 5 + r = Reader(BitArray("0x001122")) + first = r.read("u8") + r.bits.append("0xff") + +The reader position is independent of the wrapped bitstring. Mutating +``r.bits`` does not automatically adjust :attr:`Reader.pos`. + +Operations that used the stream's current position should now pass +``r.pos`` explicitly and then update it if needed:: + + # bitstring 4 + s = BitStream("0x001122") + s.pos = 8 + s.insert("0xff") + + # bitstring 5 + r = Reader(BitArray("0x001122"), pos=8) + inserted = Bits("0xff") + r.bits.insert(inserted, r.pos) + r.pos += len(inserted) + +``Reader.pos`` is deliberately lax. Assigning to :attr:`~Reader.pos`, +:attr:`~Reader.bitpos` or :attr:`~Reader.bytepos` stores the integer value +without checking it against the current length. A later read or search will +raise an error if the position cannot be used. + +Update pack() usage +=================== + +:func:`pack` now returns :class:`Bits`. In version 4 it returned ``BitStream``, +so code that immediately read from or mutated the result needs to be updated. + +For reading, wrap the result in :class:`Reader`:: + + # bitstring 4 + s = pack("uint8, uint8", 1, 2) + first = s.read("uint8") + + # bitstring 5 + bits = pack("u8, u8", 1, 2) + r = Reader(bits) + first = r.read("u8") + +For mutation, convert the result to :class:`BitArray`:: + + # bitstring 4 + s = pack("uint8", 1) + s.append("0xff") + + # bitstring 5 + s = pack("u8", 1).to_bitarray() + s.append("0xff") + +Update find() and rfind() checks +================================ + +:meth:`Bits.find`, :meth:`Bits.rfind`, :meth:`Reader.find` and +:meth:`Reader.rfind` now return ``int | None``. In version 4 they returned a +single-item tuple for success and an empty tuple for failure. + +This means a match at bit position zero evaluates as ``False`` if tested +directly. Test explicitly against ``None``:: + + # bitstring 4 + found = s.find("0xff") + if found: + pos = found[0] + + # bitstring 5 + pos = s.find("0xff") + if pos is not None: + ... + +If you used stream searching to move the current position, use +:class:`Reader`:: + + # bitstring 4 + if s.find("0xff", start=s.pos): + print(s.pos) + + # bitstring 5 + if r.find("0xff", start=r.pos) is not None: + print(r.pos) + +Use full names for bin, oct and hex +=================================== + +The ``b``, ``o`` and ``h`` aliases have been removed. Use ``bin``, ``oct`` and +``hex`` instead. The short numeric names ``u``, ``i`` and ``f`` remain. + +:: + + # bitstring 4 + data = s.h + bits = s.unpack("b12, u8") + + # bitstring 5 + data = s.hex + bits = s.unpack("bin12, u8") + +Prefer u, i and f +================= + +The bit-wise big-endian numeric dtype and keyword-initialiser names are now +``u``, ``i`` and ``f``. +The longer ``uint``, ``int`` and ``float`` names still work as compatibility +aliases, but :class:`Dtype` stringification, :class:`Array` representations and +pretty-print headers use the shorter names. + +:: + + # bitstring 4 + bits = Bits(uint=3, length=8) + dtype = Dtype("uint12") + a = Array("float16", [1.5, 2.5]) + value = r.read("int8") + + # bitstring 5 + bits = Bits(u=3, length=8) + dtype = Dtype("u12") + a = Array("f16", [1.5, 2.5]) + value = r.read("i8") + +Prefer short endian names +========================= + +The endian-specific numeric dtype and keyword-initialiser names now use the +same short base names: + +.. list-table:: + :header-rows: 1 + + * - Preferred version 5 spelling + - Compatibility alias + * - ``ube`` + - ``uintbe`` + * - ``ule`` + - ``uintle`` + * - ``une`` + - ``uintne`` + * - ``ibe`` + - ``intbe`` + * - ``ile`` + - ``intle`` + * - ``ine`` + - ``intne`` + * - ``fle`` + - ``floatle`` + * - ``fne`` + - ``floatne`` + +The plain ``f`` dtype is already big-endian, so both ``fbe`` and ``floatbe`` +are compatibility aliases for ``f``. + +:class:`Dtype` stringification, :class:`Array` representations and +pretty-print headers use the preferred names:: + + # bitstring 4 style, still accepted in bitstring 5 + bits = Bits(uintle=1, length=16) + dtype = Dtype("intbe16") + value = r.read("floatne32") + + # preferred in bitstring 5 + bits = Bits(ule=1, length=16) + dtype = Dtype("ibe16") + value = r.read("fne32") + +Use explicit construction helpers +================================= + +Some constructor forms that relied on the type of the first positional +argument have been removed or should be avoided. Use the explicit factory +methods instead. + +For zero-filled bitstrings, replace integer construction with +:meth:`Bits.from_zeros` or :meth:`BitArray.from_zeros`:: + + # bitstring 4 + a = Bits(100) + b = BitArray(100) + + # bitstring 5 + a = Bits.from_zeros(100) + b = BitArray.from_zeros(100) + +Prefer :meth:`Bits.from_string` or :meth:`BitArray.from_string` over the old +``fromstring`` spelling. The old spelling still works as a compatibility +alias.:: + + # bitstring 4 + s = Bits.fromstring("uint16=1000") + t = BitArray.fromstring("0xff") + + # bitstring 5 + s = Bits.from_string("u16=1000") + t = BitArray.from_string("0xff") + +Use the other factory methods for construction from values that are no longer +accepted as the first positional argument:: + + # bitstring 4 + a = Bits([1, 0, 1]) + f = open("data.bin", "rb") + b = Bits(f) + f.close() + c = Bits(io.BytesIO(b"\x01\x02")) + d = Bits(array_obj) + + # bitstring 5 + a = Bits.from_bools([1, 0, 1]) + with open("data.bin", "rb") as f: + b = Bits.from_file(f) + c = Bits.from_bytes(b"\x01\x02") + d = Bits.from_bytes(array_obj.tobytes()) + +If you used the external ``bitarray`` package, convert explicitly. For exact +bit lengths, :meth:`Bits.from_bools` is the most direct replacement. For +byte-oriented data, use :meth:`Bits.from_bytes` with an explicit length:: + + bits = Bits.from_bools(bitarray_obj) + bits = Bits.from_bytes(bitarray_obj.tobytes(), length=len(bitarray_obj)) + +The old ``tobitarray()`` method returned an object from the external +``bitarray`` package and has been removed. :meth:`Bits.to_bitarray` returns a +bitstring :class:`BitArray` instead. If you still need an external +``bitarray`` object, create it explicitly using that package's API, for example +from the bitstring as an iterable of booleans or from :meth:`Bits.to_bytes`. + +The ``filename=`` keyword constructor still exists, but new code should prefer +:meth:`Bits.from_file` or :meth:`BitArray.from_file`:: + + # preferred in bitstring 5 + s = Bits.from_file("data.bin", offset=8, length=32) + +Make optional range arguments keyword-only +========================================== + +Several methods now require optional range and search arguments to be named. +This makes calls harder to misread and leaves the first positional arguments +for the data being operated on. + +Update calls like this:: + + # bitstring 4 + s.find("0xff", 8, 64) + s.findall("0b1", 0, 100, 3) + s.split("0x00", 8) + a.replace("0b0", "0b1", 4, 20) + a.reverse(8, 24) + + # bitstring 5 + s.find("0xff", start=8, end=64) + s.findall("0b1", start=0, end=100, count=3) + s.split("0x00", start=8) + a.replace("0b0", "0b1", start=4, end=20) + a.reverse(start=8, end=24) + +The affected methods are :meth:`Bits.cut`, :meth:`Bits.find`, +:meth:`Bits.findall`, :meth:`Bits.rfind`, :meth:`Bits.split`, +:meth:`BitArray.replace`, :meth:`BitArray.reverse`, :meth:`BitArray.rol` and +:meth:`BitArray.ror`. + +Rename Dtype build/parse +======================== + +``Dtype.build`` and ``Dtype.parse`` have been renamed to +:meth:`Dtype.pack` and :meth:`Dtype.unpack`:: + + # bitstring 4 + d = Dtype("uint8") + bits = d.build(42) + value = d.parse(bits) + + # bitstring 5 + d = Dtype("u8") + bits = d.pack(42) + value = d.unpack(bits) + +Remove LSB0 mode +================ + +The old LSB0 mode has been removed. There is no +``bitstring.options.lsb0`` setting in version 5, and indexing is always MSB0. + +Code that enabled LSB0 mode needs to translate positions explicitly. For a +single bit position ``i`` in a bitstring ``s``, the equivalent MSB0 position is +``len(s) - 1 - i``. Slice translations depend on the direction and bounds of +the original slice, so they should be reviewed case by case. + +Use explicit MXFP overflow dtypes +================================= + +The ``bitstring.options.mxfp_overflow`` setting has been removed. The overflow +policy for OCP MXFP E4M3 and E5M2 packing is now part of the dtype name, and +the old unsuffixed ``e4m3mxfp`` and ``e5m2mxfp`` names have been removed. + +Use ``e4m3mxfp_saturate`` or ``e5m2mxfp_saturate`` for saturating behaviour. +Use ``e4m3mxfp_overflow`` or ``e5m2mxfp_overflow`` for the old overflow mode:: + + # bitstring 4 + bitstring.options.mxfp_overflow = "overflow" + bits = Bits(e4m3mxfp=1e10) + + # bitstring 5 + bits = Bits(e4m3mxfp_overflow=1e10) + +Code that used the default saturating behaviour also needs to choose it +explicitly:: + + # bitstring 4 + bits = Bits(e5m2mxfp=1e10) + + # bitstring 5 + bits = Bits(e5m2mxfp_saturate=1e10) + +Use options for module settings +=============================== + +The old module-level option variables have been removed. Set options through +the :data:`bitstring.options` object:: + + # bitstring 4 + bitstring.bytealigned = True + + # bitstring 5 + bitstring.options.bytealigned = True + +Prefer the new underscored method names +======================================= + +Version 5 adds underscored spellings for several older method names. The old +names remain as compatibility aliases, but new code and documentation should +use the underscored names. + +.. list-table:: + :header-rows: 1 + + * - Version 4 spelling + - Preferred version 5 spelling + * - ``tobytes()`` + - :meth:`Bits.to_bytes` + * - ``tofile(f)`` + - :meth:`Bits.to_file` + * - ``tolist()`` + - :meth:`Array.to_list` + * - ``fromfile(f)`` + - :meth:`Array.from_file` + * - ``readlist(fmt)`` + - :meth:`Reader.read_list` + * - ``peeklist(fmt)`` + - :meth:`Reader.peek_list` + * - ``readto(bs)`` + - :meth:`Reader.read_to` + * - ``bytealign()`` + - :meth:`Reader.byte_align` + +Remove command-line usage +========================= + +The ``python -m bitstring`` command-line interface has been removed. Use a +small Python script or shell one-liner for any remaining uses. + +Suggested upgrade order +======================= + +For a large codebase, the least surprising order is: + +1. Update imports to remove ``ConstBitStream`` and ``BitStream``. +2. Introduce :class:`Reader` wherever code uses ``pos``, ``read``, ``peek`` or + stream-style searching. +3. Update :func:`pack` call sites that relied on the old ``BitStream`` return + value. +4. Change ``find`` and ``rfind`` checks to use ``is not None``. +5. Replace ``b``, ``o`` and ``h`` aliases with ``bin``, ``oct`` and ``hex``. +6. Prefer ``u``, ``i`` and ``f`` over ``uint``, ``int`` and ``float``. +7. Prefer short endian names such as ``ule`` and ``ibe`` over ``uintle`` and + ``intbe``. +8. Replace removed constructor forms with explicit factory methods. +9. Add keywords to optional range and search arguments. +10. Rename ``Dtype.build`` / ``Dtype.parse`` and remove any LSB0 usage. +11. Replace any ``options.mxfp_overflow`` changes with explicit MXFP dtype + names. +12. Optionally update compatibility aliases such as ``tobytes`` and + ``readlist`` to their preferred underscored names. diff --git a/doc/walkthrough.ipynb b/doc/walkthrough.ipynb index c29c6460..c6889325 100644 --- a/doc/walkthrough.ipynb +++ b/doc/walkthrough.ipynb @@ -58,8 +58,8 @@ "source": [ "\n", "We've imported the `BitArray` class directly to save some typing.\n", - "This is just a mutable container for binary data. Later on we'll also use the `BitStream` class which adds a bit position and reading methods to treat the data as a stream.\n", - "There are also immutable versions of both these classes that we won't be using here.\n", + "This is a mutable container for binary data. Later on we'll also use `Reader`, which wraps a bitstring with a bit position and reading methods to treat the data as a stream.\n", + "There is also an immutable `Bits` class that is useful when the binary data doesn't need to be modified.\n", "\n", "We can now create a couple of bitstrings:" ] @@ -89,8 +89,8 @@ "Note:\n", "\n", "Be sure to remember the quotes around the hex and binary strings.\n", - "If you forget them you would just have an ordinary Python integer, which would instead create a bitstring of that many '0' bits.\n", - "For example `0xff01` is the same as the base-10 number 65281, so `BitArray(0xff01)` would consist of 65281 zero bits!\n", + "If you forget them you would just have an ordinary Python integer, which bitstring rejects as an ambiguous initializer.\n", + "For example `0xff01` is the same as the base-10 number 65281, so write `BitArray('0xff01')` for those 16 bits or `BitArray.from_zeros(0xff01)` if you really want 65281 zero bits.\n", "\n", "\n", "There are lots of things we can do with our new bitstrings, the simplest of which is just to print them:" @@ -145,7 +145,7 @@ "source": [ "print(a.bin)\n", "print(b.oct)\n", - "print(b.int)\n", + "print(b.i)\n", "print(a.bytes)" ] }, @@ -156,8 +156,8 @@ "\n", "There are a few things to note here:\n", "\n", - "* To get the different interpretations of the binary data we use properties such as `bin`, `hex`, `oct`, `int` and `bytes`. You can probably guess what these all mean, but you don't need to know quite yet. The properties are calculated when you ask for them rather than being stored as part of the object itself.\n", - "* Many of the interpretations have single letter aliases, and interpretations can also have bit lengths appended to them. This allows expressions such as `a.u32 = 900` which will set `a` to the 32 bit representation of the unsigned integer `900`. You're not restricted to the usual bit lengths, so something like `a.i5 = -8` will work as well.\n", + "* To get the different interpretations of the binary data we use properties such as `bin`, `hex`, `oct`, `i`, `u` and `bytes`. You can probably guess what these all mean, but you don't need to know quite yet. The properties are calculated when you ask for them rather than being stored as part of the object itself.\n", + "* The numeric interpretations use short names, and interpretations can also have bit lengths appended to them. This allows expressions such as `a.u32 = 900` which will set `a` to the 32 bit representation of the unsigned integer `900`. You're not restricted to the usual bit lengths, so something like `a.i5 = -8` will work as well.\n", "\n", "Great - let's try some more:" ] @@ -223,7 +223,7 @@ "source": [ "\n", "Here we've added a zero bit first to the end and then to the start.\n", - "Don't worry too much about how it all works, but just to give you a taster the zero bit `[0]` could also have been written as `BitArray([0])`, `BitArray('0b0')`, `BitArray(bin='0')`, `'0b0'` or just `1` (this final option isn't a typo, it means construct a bitstring of length one, with all the bits initialised to zero - it does look a bit confusing though which is why I prefer `[0]` and `[1]` to represent single bits).\n", + "Don't worry too much about how it all works, but just to give you a taster the zero bit `[0]` could also have been written as `BitArray.from_bools([0])`, `BitArray('0b0')`, `BitArray(bin='0')` or `'0b0'`.\n", "Take a look at [the auto initialiser](https://bitstring.readthedocs.io/en/stable/creation.html#the-auto-initialiser) for more details." ] }, @@ -338,7 +338,7 @@ "outputs": [ { "data": { - "text/plain": "(3,)" + "text/plain": "3" }, "execution_count": 10, "metadata": {}, @@ -434,8 +434,8 @@ "source": [ "s = BitArray()\n", "s.append('0x000001b3') # the sequence_header_code\n", - "s.append('uint12=352') # 12 bit unsigned integer\n", - "s.append('uint12=288')\n", + "s.append('u12=352') # 12 bit unsigned integer\n", + "s.append('u12=288')\n", "print(s.hex)" ] }, @@ -452,7 +452,7 @@ "metadata": {}, "outputs": [], "source": [ - "s = BitArray('0x000001b3, uint12=352, uint12=288')" + "s = BitArray('0x000001b3, u12=352, u12=288')" ] }, { @@ -477,7 +477,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "where we have also used `2*u12` as shorthand for `uint12, uint12`.\n", + "where we have also used `2*u12` as shorthand for `u12, u12`.\n", "\n", "The [`pack`](https://bitstring.readthedocs.io/en/stable/functions.html#bitstring.pack) function can also take a dictionary as a parameter which can replace the tokens in the format string.\n", "For example:" @@ -490,7 +490,7 @@ "outputs": [ { "data": { - "text/plain": "\"BitStream('0x000001b31601201')\"" + "text/plain": "\"Bits('0x000001b31601201')\"" }, "execution_count": 15, "metadata": {}, @@ -499,9 +499,9 @@ ], "source": [ "fmt = 'sequence_header_code, \\\n", - " uint12=horizontal_size_value, \\\n", - " uint12=vertical_size_value, \\\n", - " uint4=aspect_ratio_information, '\n", + " u12=horizontal_size_value, \\\n", + " u12=vertical_size_value, \\\n", + " u4=aspect_ratio_information, '\n", " # ...\n", "d = {'sequence_header_code': '0x000001b3',\n", " 'horizontal_size_value': 352,\n", @@ -518,12 +518,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Parsing bitstreams\n", + "## Reading sequentially\n", "\n", - "You might have noticed that `pack` returned a `BitStream` rather than a `BitArray`.\n", - "This isn't a problem as the `BitStream` class just adds a few stream-like qualities to `BitArray` which we'll take a quick look at here.\n", + "`pack` returns an immutable `Bits` object.\n", + "To read it sequentially, wrap it in a `Reader`.\n", "\n", - "The stream-ness of this object is via its bit position, and various reading and peeking methods.\n", + "The stream-ness is provided by the reader's bit position, and various reading and peeking methods.\n", "First let's try a read or two, and see how this affects the bit position:\n" ] }, @@ -535,9 +535,9 @@ { "name": "stdout", "output_type": "stream", - "text": [ + "text": [ "0\n", - "BitStream('0x000001')\n", + "Bits('0x000001')\n", "24\n", "b3\n", "32\n" @@ -545,18 +545,19 @@ } ], "source": [ - "print(s.pos)\n", - "print(repr(s.read(24)))\n", - "print(s.pos)\n", - "print(s.read('hex8'))\n", - "print(s.pos)" + "r = bitstring.Reader(s)\n", + "print(r.pos)\n", + "print(repr(r.read(24)))\n", + "print(r.pos)\n", + "print(r.read('hex8'))\n", + "print(r.pos)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "First we read 24 bits, which returned a new `BitStream` object, then we used a format string to read 8 bits interpreted as a hexadecimal string.\n", + "First we read 24 bits, which returned a new `Bits` object, then we used a format string to read 8 bits interpreted as a hexadecimal string.\n", "We know that the next two sets of 12 bits were created from integers, so to read them back we can say" ] }, @@ -575,7 +576,7 @@ } ], "source": [ - "s.readlist('2*u12')" + "r.read_list('2*u12')" ] }, { @@ -583,8 +584,8 @@ "metadata": {}, "source": [ "\n", - "If you don't want to use a bitstream then you can always use [`unpack`](https://bitstring.readthedocs.io/en/stable/bits.html#bitstring.Bits.unpack).\n", - "This takes much the same form as [`readlist`](https://bitstring.readthedocs.io/en/stable/constbitstream.html#bitstring.ConstBitStream.readlist) except it just unpacks from the start of the bitstring.\n", + "If you don't need a `Reader` then you can always use [`unpack`](https://bitstring.readthedocs.io/en/stable/bits.html#bitstring.Bits.unpack).\n", + "This takes much the same form as [`read_list`](https://bitstring.readthedocs.io/en/stable/reader.html#bitstring.Reader.read_list) except it just unpacks from the start of the bitstring.\n", "For example:" ] }, @@ -603,7 +604,7 @@ } ], "source": [ - "s.unpack('bytes4, 2*u12, uint4')" + "s.unpack('bytes4, 2*u12, u4')" ] }, { @@ -688,7 +689,7 @@ "source": [ "# Create a BitArray with a hundred million 'one' bits\n", "limit = 100_000_000\n", - "is_prime = BitArray(limit)\n", + "is_prime = BitArray.from_zeros(limit)\n", "is_prime.set(True)\n", "# Manually set 0 and 1 to be not prime.\n", "is_prime.set(False, [0, 1])\n", @@ -698,7 +699,7 @@ " is_prime.set(False, range(i*i, limit, i))\n", "\n", "print(f\"There are {is_prime.count(True)} primes less than {limit},\")\n", - "print(f\"the largest one of which is {is_prime.rfind('0b1')[0]}\")\n", + "print(f\"the largest one of which is {is_prime.rfind('0b1')}\")\n", "print(f\"and there are {len(list(is_prime.findall('0b101')))} twin primes.\")" ] }, diff --git a/pyproject.toml b/pyproject.toml index 64b27c41..0b9a6621 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,21 +4,19 @@ build-backend = "setuptools.build_meta" [project] name = "bitstring" -version = "4.4.0" +version = "5.0.0_beta1" license = { text = "MIT" } authors = [ { name="Scott Griffiths", email="dr.scottgriffiths@gmail.com" }, ] description = "Simple construction, analysis and modification of binary data." readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.10" classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -29,8 +27,7 @@ classifiers = [ ] keywords = ["binary", "bitarray", "bitvector", "bitfield"] dependencies = [ - "bitarray >= 3.0.0, < 4.0", - "tibs >= 0.5.6, < 0.6", + "tibs >= 0.12.0, <= 2.0", ] [project.urls] diff --git a/release_notes.md b/release_notes.md index 203f954e..f32558ce 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,6 +1,105 @@ # Release Notes +### Upcoming: version 5.0 + +This version completes the move to the `tibs` Rust library as the core bit storage +and manipulation layer. The optional Rust backend introduced in 4.4 is now the only +backend, which removes a lot of compatibility code and should allow more optimisation +work from here. + +Breaking changes: + +* Removed the `ConstBitStream` and `BitStream` classes. Use `Reader(Bits(...))` + for immutable sequential reading, or `Reader(BitArray(...))` when the wrapped + bitstring also needs to be mutated. +* Dropped support for Python 3.8 and 3.9. The minimum supported version is now + Python 3.10. +* Removed `bitarray` library compatibility from the public API. Bitstrings can no longer be + initialised directly from `bitarray` objects, the `bitarray=` keyword initialiser + has been removed, and the `tobitarray()` method has been removed. +* Removed LSB0 mode. This removes `bitstring.options.lsb0` and all LSB0-specific + indexing, slicing, reading, packing, unpacking and pretty-printing behaviour. +* `pack()` now returns a `Bits` object instead of a `BitStream`. +* `find()` and `rfind()` now return `int | None` instead of `(pos,)` or `()`. + Use `result is not None` when testing whether a search succeeded, as bit + position zero is a valid match. +* Removed the single-letter `b`, `o` and `h` aliases for `bin`, `oct` and + `hex`. The numeric names `u`, `i` and `f` remain and are now canonical. +* Made `u`, `i` and `f` the canonical dtype names for bit-wise big-endian + unsigned integers, signed integers and floats. The longer `uint`, `int` and + `float` names remain as compatibility aliases for dtype tokens and keyword + initialisers, but dtype stringification, `Array` representations and + pretty-print headers now use the short forms. +* Made `ube`, `ule`, `une`, `ibe`, `ile`, `ine`, `fle` and `fne` the canonical + endian-specific dtype and keyword-initialiser names. The longer names such as + `uintle`, `intbe` and `floatne` remain as compatibility aliases. +* Positional integer construction has been removed. Use `Bits.from_zeros(100)` + or `BitArray.from_zeros(100)` instead of `Bits(100)` or `BitArray(100)`. +* Direct construction from arbitrary boolean iterables, file objects, + `io.BytesIO`, and `array.array` objects has been removed. Use + `from_bools()`, `from_file()` or `from_bytes()` instead. +* `Dtype.build()` and `Dtype.parse()` have been renamed to `Dtype.pack()` and + `Dtype.unpack()`. +* Optional range/search arguments such as `start`, `end`, `count` and + `bytealigned` are now keyword-only for search, split, cut, replace and + reverse/rotate-style methods. +* Removed the `bitstring.options.mxfp_overflow` global setting and the + unsuffixed `e4m3mxfp` and `e5m2mxfp` dtypes. Use explicit + `e4m3mxfp_saturate`, `e4m3mxfp_overflow`, `e5m2mxfp_saturate` or + `e5m2mxfp_overflow` dtypes instead. +* Removed the `python -m bitstring` command-line interface. +* Removed the deprecated module-level way of setting options. Use + `bitstring.options.bytealigned` rather than `bitstring.bytealigned`. + +Other changes and fixes: + +* Removed the `bitarray` dependency. The `tibs` dependency is now required. +* Removed the optional backend selection mechanism. The `BITSTRING_USE_RUST_CORE` + environment variable and `bitstring.options.using_rust_core` flag no longer exist. +* Added faster direct read paths for many dtypes, avoiding temporary bitstring + allocation when reading through `Reader`. +* Added the `Reader` class, which wraps a `Bits` or `BitArray` object and stores + an independent bit position for `read`, `read_list`, `peek`, `peek_list`, + `read_to`, `find`, `rfind` and `byte_align`. +* Added explicit construction helpers: `from_string()`, `from_dtype()`, + `from_bytes()`, `from_bools()`, `from_zeros()`, `from_ones()`, + `from_joined()` and `from_file()`. +* Added `from_tibs()` and `to_tibs()` interop helpers for converting between + bitstring types and the lower-level `tibs.Tibs` type. `tibs.Tibs` and + `tibs.Mutibs` are now counted as bitstring-like inputs. +* Added explicit saturate and overflow dtype variants for the OCP MXFP E4M3 + and E5M2 formats. +* Several method names now have underscored preferred spellings: + `to_bytes()`, `to_file()`, `to_list()`, `from_string()`, `from_file()`, + `read_list()`, `peek_list()`, `read_to()` and `byte_align()`. The old + spellings without underscores remain as compatibility aliases. +* Added `Bits.to_bitarray()` and `BitArray.to_bits()` conversion methods. +* Improved `from_bytes()`, `from_file()` and keyword construction from bytes, + memoryviews and files with offsets and lengths by using tibs offset/length + support directly. +* File-backed bitstrings with a non-zero offset no longer have to read the whole file + into memory. +* Added a faster `cut()` path for immutable bitstrings. +* Improved string-token creation with a fast path for literal-only strings such as + `0xff, 0b101, 0o7`. +* Fixed signed and unsigned integer interpretation for very large bitstrings that are + wider than tibs' native integer conversion range. +* Fixed `Array.from_file()` to honour the file object's current position, to reject + negative item counts, and to report short reads correctly. +* Fixed `Array.insert()` clamping for very negative indices to match `list.insert`. +* Fixed in-place `Array` arithmetic and bitwise operations with another `Array` so + they update and return the original object. +* Fixed scaled `Dtype` equality and hashing so dtypes with different scale factors + compare as distinct. +* Fixed parsing of repeated token groups so `0*(...)` is allowed and negative repeat + factors are rejected. +* Fixed repeated construction of `Options()` so it no longer resets existing option + values. +* Fixed bitstore equality comparisons against unrelated object types. +* Added more regression tests around the tibs-backed bitstore, `Array` operations, + dtype scaling, token parsing and option handling. + ### March 2026: version 4.4.0 This version adds a new optional Rust-based backend. This is turned off by default so diff --git a/tests/test_array.py b/tests/test_array.py index f1e974c7..06e72230 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -95,7 +95,7 @@ def test_creation_with_trailing_bits(self): b = Array('bin:3', ['111', '000', '111']) assert len(b) == 3 assert b.data == '0b111000111' - b.dtype = 'h4' + b.dtype = 'hex4' assert len(b) == 2 with pytest.raises(ValueError): b.append('f') @@ -136,7 +136,7 @@ def test_creation_from_bits(self): def test_creation_from_array_array(self): a = array.array('H', [10, 20, 30, 40]) - b = Array('uintne16', a) + b = Array('une16', a) assert a.tolist() == b.tolist() assert a.tobytes() == b.tobytes() with pytest.raises(ValueError): @@ -152,9 +152,9 @@ def test_different_type_codes(self): a = Array('>H', [10, 20]) assert a.data.unpack('2*uint16') == a.tolist() a = Array(' [ + assert remove_unprintable(s.getvalue()) == """ [ 0: 12 100 99 ]\n""" @@ -565,8 +585,8 @@ def test_pp_two_formats(self): def test_pp_two_formats_no_length(self): a = Array('float16', bytearray(range(50, 56))) s = io.StringIO() - a.pp(stream=s, fmt='u, b') - assert remove_unprintable(s.getvalue()) == """ [ + a.pp(stream=s, fmt='u, bin') + assert remove_unprintable(s.getvalue()) == """ [ 0: 12851 13365 13879 : 0011001000110011 0011010000110101 0011011000110111 ]\n""" @@ -731,7 +751,7 @@ def test_neg(self): a = Array('i92', [-1, 1, 0, 100, -100]) b = -a assert b.tolist() == [1, -1, 0, -100, 100] - assert str(b.dtype) == 'int92' + assert str(b.dtype) == 'i92' def test_abs(self): a = Array('float16', [-2.0, 0, -0, 100, -5.5]) @@ -748,7 +768,7 @@ def test_appending_auto(self): assert a[0] == Bits('0xff') with pytest.raises(TypeError): a += 8 - a.append(Bits(8)) + a.append(Bits.from_zeros(8)) assert a[:].equals(Array('bits:8', ['0b1111 1111', Bits('0x00')])) a.extend(['0b10101011']) assert a[-1].hex == 'ab' @@ -913,9 +933,20 @@ def test_set_out_of_range_element(self): def test_bytes(self): a = Array('bytes8', 5) assert a.data == b'\x00'*40 + assert len(a) == 5 + assert a.itemsize == 64 b = Array('bytes1', 5) assert b.data == b'\x00'*5 + assert len(b) == 5 + assert b.itemsize == 8 + + def test_bytes_elements_from_iterable(self): + a = Array('bytes3', [b'ABC', b'DEF', b'ZZZ']) + assert len(a) == 3 + assert a.itemsize == 24 + assert a.to_list() == [b'ABC', b'DEF', b'ZZZ'] + assert a[1] == b'DEF' def test_bytes_trailing_bits(self): b = Bits('0x000000, 0b111') @@ -926,4 +957,4 @@ def test_operation_with_bool(self): x = Array('int4', [1, 2, 3, 4]) y = Array('float16', [100, 2.0, 0.0, 4]) x = x + (y == 0.0) - assert x.tolist() == [1, 2, 4, 4] \ No newline at end of file + assert x.tolist() == [1, 2, 4, 4] diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index d80e7a55..3acce995 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -19,7 +19,7 @@ def cut(): def test_count(benchmark): def count(): - s = bitstring.BitArray(100000000) + s = bitstring.BitArray.from_zeros(100000000) s.set(1, [10, 100, 1000, 10000000]) return s.count(1) c = benchmark(count) @@ -32,7 +32,7 @@ def token_parsing(): s += 'uint:12=244, float:32=0.4' s += '0x3e44f, 0b11011, 0o75523' s += [0, 1, 2, 0, 0, 1, 2, 0, -1, 0, 'hello'] - s += bitstring.BitArray(104) + s += bitstring.BitArray.from_zeros(104) benchmark(token_parsing) def test_findall(benchmark): @@ -50,7 +50,7 @@ def test_repeated_reading(benchmark): def repeating_reading(): random.seed(1414) i = int.to_bytes(random.getrandbits(100000*8), 100000, 'big') - s = bitstring.ConstBitStream(bytes=i) + s = bitstring.Reader(bitstring.Bits(bytes=i)) for _ in range(800000 // 40): _ = s.readlist('uint:4, float:32, bool, bool, bool, bool') benchmark(repeating_reading) @@ -58,7 +58,7 @@ def repeating_reading(): def test_primes(benchmark): def primes(): limit = 1000000 - is_prime = bitstring.BitArray(limit) + is_prime = bitstring.BitArray.from_zeros(limit) is_prime.set(True) # Manually set 0 and 1 to be not prime. is_prime.set(False, [0, 1]) diff --git a/tests/test_bitarray.py b/tests/test_bitarray.py index 2e4d5fa0..6fe0bc26 100644 --- a/tests/test_bitarray.py +++ b/tests/test_bitarray.py @@ -6,18 +6,109 @@ import pytest import sys import os -import bitarray import bitstring from bitstring import BitArray, Bits +from tibs import Mutibs, Tibs sys.path.insert(0, '..') class TestAll: - def test_creation_from_uint(self): - s = BitArray(uint=15, length=6) + def test_explicit_factory_methods_return_bitarray(self, tmp_path): + assert type(BitArray.from_string("0xf")) is BitArray + assert type(BitArray.from_dtype("u8", 12)) is BitArray + assert type(BitArray.from_bytes(b"\xf0", offset=1, length=3)) is BitArray + assert type(BitArray.from_bools([True, 0, "x"])) is BitArray + assert type(BitArray.from_zeros(5)) is BitArray + assert type(BitArray.from_ones(5)) is BitArray + assert type(BitArray.from_joined(["0xa", "0xb"])) is BitArray + assert BitArray.from_joined(["0xa", "0xb"]) == "0xab" + assert BitArray.from_joined([]) == BitArray() + + filename = tmp_path / "factory.bin" + filename.write_bytes(b"\x12\x34") + assert type(BitArray.from_file(filename)) is BitArray + + bits = BitArray.from_ones(4) + bits.append("0b0") + assert bits == "0b11110" + + def test_to_bits(self): + bitarray = BitArray("0b101") + bits = bitarray.to_bits() + assert type(bits) is Bits + assert bits == bitarray + bitarray.append("0b1") + assert bits == "0b101" + assert bitarray == "0b1011" + + def test_tibs_conversion(self): + tibs = Tibs.from_bin("101") + bitarray = BitArray.from_tibs(tibs) + assert type(bitarray) is BitArray + assert bitarray == "0b101" + assert type(bitarray.to_tibs()) is Tibs + assert bitarray.to_tibs() == tibs + + snapshot = bitarray.to_tibs() + bitarray.append("0b0") + assert snapshot == tibs + assert bitarray == "0b1010" + + mutibs = Mutibs.from_bin("101") + bitarray = BitArray.from_tibs(mutibs) + assert type(bitarray) is BitArray + assert bitarray == "0b101" + mutibs.append(0) + assert bitarray == "0b101" + + @pytest.mark.parametrize("tibs_type", [Tibs, Mutibs]) + def test_constructor_accepts_tibs_types(self, tibs_type): + tibs = tibs_type.from_bin("101") + bitarray = BitArray(tibs) + assert type(bitarray) is BitArray + assert bitarray == "0b101" + + bitarray.append("0b0") + assert bitarray == "0b1010" + + @pytest.mark.parametrize("tibs_type", [Tibs, Mutibs]) + def test_constructor_rejects_tibs_types_with_length_or_offset(self, tibs_type): + tibs = tibs_type.from_bin("101") + with pytest.raises(bitstring.CreationError, match="explicit lengths or offsets"): + BitArray(tibs, length=2) + with pytest.raises(bitstring.CreationError, match="explicit lengths or offsets"): + BitArray(tibs, offset=1) + + def test_mutibs_input_is_copied(self): + source = Mutibs.from_bin("101") + constructed = BitArray(source) + from_tibs = BitArray.from_tibs(source) + + source.append(0) + + assert constructed == "0b101" + assert from_tibs == "0b101" + + constructed.append("0b1") + assert source.to_bin() == "1010" + + def test_tibs_types_promote_as_bits_type(self): + bitarray = BitArray("0b10") + bitarray.append(Tibs.from_bin("11")) + assert bitarray == "0b1011" + + bitarray.append(Mutibs.from_bin("00")) + assert bitarray == "0b101100" + + replacements = bitarray.replace(Mutibs.from_bin("0"), Tibs.from_bin("11"), count=1) + assert replacements == 1 + assert bitarray == "0b1111100" + + def test_creation_from_u(self): + s = BitArray(u=15, length=6) assert s.bin == '001111' - s = BitArray(uint=0, length=1) + s = BitArray(u=0, length=1) assert s.bin == '0' s.u = 1 assert s.uint == 1 @@ -73,7 +164,7 @@ def test_insert_self(self): assert a == '0b1111110000' def test_no_bit_pos_for_insert(self): - s = BitArray(100) + s = BitArray.from_zeros(100) with pytest.raises(TypeError): s.insert('0xabc') @@ -124,7 +215,7 @@ def test_set_item(self): class TestBugs: def test_adding_nonsense(self): - a = BitArray([0]) + a = BitArray.from_bools([0]) with pytest.raises(ValueError): a += '3' with pytest.raises(ValueError): @@ -145,12 +236,6 @@ def test_prepend_after_creation_from_data_with_offset(self): class TestByteAligned: - def test_changing_it(self): - bitstring.bytealigned = True - assert bitstring.bytealigned - bitstring.bytealigned = False - assert not bitstring.bytealigned - def test_not_byte_aligned(self): a = BitArray('0xff00ff') s = a.split('0xff') @@ -159,9 +244,9 @@ def test_not_byte_aligned(self): a = BitArray('0x00 ff 0f f') li = list(a.findall('0xff')) assert li == [8, 20] - p = a.find('0x0f')[0] + p = a.find('0x0f') assert p == 4 - p = a.rfind('0xff')[0] + p = a.rfind('0xff') assert p == 20 s = list(a.split('0xff')) assert s == ['0x00', '0xff0', '0xff'] @@ -169,19 +254,20 @@ def test_not_byte_aligned(self): assert a == '0x000' def test_byte_aligned(self): - bitstring.bytealigned = True + # TODO: This really should be done with a context. + bitstring.options.bytealigned = True a = BitArray('0x00 ff 0f f') li = list(a.findall('0xff')) assert li == [8] - p = a.find('0x0f')[0] + p = a.find('0x0f') assert p == 16 - p = a.rfind('0xff')[0] + p = a.rfind('0xff') assert p == 8 s = list(a.split('0xff')) assert s == ['0x00', '0xff0ff'] a.replace('0xff', '') assert a == '0x000ff' - bitstring.bytealigned = False + bitstring.options.bytealigned = False class TestSliceAssignment: @@ -213,7 +299,7 @@ def test_slice_assignment_muliple_bits(self): a[0] = '0b000' assert a.bin5 == '00010' a[0:3] = '0b111' - assert a.b5 == '11110' + assert a.bin5 == '11110' a[-2:] = '0b011' assert a.bin == '111011' a[:] = '0x12345' @@ -264,11 +350,11 @@ def test_del_slice_negative_end(self): assert a == '0b01000001' def test_del_slice_errors(self): - a = BitArray(10) + a = BitArray.from_zeros(10) del a[5:3] - assert a == Bits(10) + assert a == Bits.from_zeros(10) del a[3:5:-1] - assert a == Bits(10) + assert a == Bits.from_zeros(10) def test_del_single_element(self): a = BitArray('0b0010011') @@ -288,14 +374,14 @@ def test_set_slice_step(self): a[7:3:-1] = [1, 1, 1, 0] assert a.bin == '1010011110' a[7:1:-2] = [0, 0, 1] - assert a.b == '1011001010' + assert a.bin == '1011001010' a[::-5] = [1, 1] assert a.bin == '1011101011' a[::-1] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 1] assert a.bin == '1000000000' def test_set_slice_step_with_int(self): - a = BitArray(9) + a = BitArray.from_zeros(9) a[5:8] = -1 assert a.bin == '000001110' a[:] = 10 @@ -306,11 +392,11 @@ def test_set_slice_step_with_int(self): assert a.bin == '111110101' def test_set_slice_errors(self): - a = BitArray(8) + a = BitArray.from_zeros(8) with pytest.raises(ValueError): a[::3] = [1] - class A(object): + class A: pass with pytest.raises(TypeError): a[1:2] = A() @@ -343,7 +429,7 @@ def test_clear(self): class TestCopy: def test_copy_method(self): - s = BitArray(9) + s = BitArray.from_zeros(9) t = s.copy() assert s == t t[0] = True @@ -354,7 +440,7 @@ def test_copy_method(self): class TestModifiedByAddingBug: def test_adding(self): - a = BitArray.fromstring('0b0') + a = BitArray.from_string('0b0') b = BitArray('0b11') c = a + b assert c == '0b011' @@ -362,283 +448,6 @@ def test_adding(self): assert b == '0b11' -class TestLsb0Setting: - - @classmethod - def setup_class(cls): - bitstring.lsb0 = True - - @classmethod - def teardown_class(cls): - bitstring.lsb0 = False - - def test_set_single_bit(self): - a = BitArray(10) - a[0] = True - assert a == '0b0000000001' - a[1] = True - assert a == '0b0000000011' - a[0] = False - assert a == '0b0000000010' - a[9] = True - assert a == '0b1000000010' - with pytest.raises(IndexError): - a[10] = True - - def test_set_single_negative_bit(self): - a = BitArray('0o000') - a[-1] = True - assert a == '0b100000000' - a[-2] = True - assert a == '0o600' - a[-9] = True - assert a == '0o601' - with pytest.raises(IndexError): - a[-10] = True - - def test_invert_bit(self): - a = BitArray('0b11110000') - a.invert() - assert a == '0x0f' - a.invert(0) - assert a == '0b00001110' - a.invert(-1) - assert a == '0b10001110' - - def test_deleting_bits(self): - a = BitArray('0b11110') - del a[0] - assert a == '0xf' - - def test_deleting_range(self): - a = BitArray('0b101111000') - del a[0:1] - assert a == '0b10111100' - del a[2:6] - assert a == '0b1000' - a = BitArray('0xabcdef') - del a[:8] - assert a == '0xabcd' - del a[-4:] - assert a == '0xbcd' - del a[:-4] - assert a == '0xb' - - def test_appending_bits(self): - a = BitArray('0b111') - a.append('0b000') - assert a.bin == '000111' - a += '0xabc' - assert a == '0xabc, 0b000111' - - def test_setting_slice(self): - a = BitArray('0x012345678') - a[4:12] = '0xfe' - assert a == '0x012345fe8' - a[0:4] = '0xbeef' - assert a == '0x012345febeef' - - def test_truncating_start(self): - a = BitArray('0b1110000') - a = a[4:] - assert a == '0b111' - - def test_truncating_end(self): - a = BitArray('0x123456') - a = a[:16] - assert a == '0x3456' - - def test_all(self): - a = BitArray('0b0000101') - assert a.all(1, [0, 2]) - assert a.all(False, [-1, -2, -3, -4]) - - b = Bits(bytes=b'\x00\xff\xff', offset=7) - assert b.all(1, [1, 2, 3, 4, 5, 6, 7]) - assert b.all(1, [-2, -3, -4, -5, -6, -7, -8]) - - def test_any(self): - a = BitArray('0b0001') - assert a.any(1, [0, 1, 2]) - - def test_endswith(self): - a = BitArray('0xdeadbeef') - assert a.endswith('0xdead') - - def test_startswith(self): - a = BitArray('0xdeadbeef') - assert a.startswith('0xbeef') - - def test_cut(self): - a = BitArray('0xff00ff1111ff2222') - li = list(a.cut(16)) - assert li == ['0x2222', '0x11ff', '0xff11', '0xff00'] - - def test_find(self): - t = BitArray('0b10') - p, = t.find('0b1') - assert p == 1 - t = BitArray('0b1010') - p, = t.find('0b1') - assert p == 1 - a = BitArray('0b10101010, 0xabcd, 0b10101010, 0x0') - p, = a.find('0b10101010', bytealigned=False) - assert p == 4 - p, = a.find('0b10101010', start=4, bytealigned=False) - assert p == 4 - p, = a.find('0b10101010', start=5, bytealigned=False) - assert p == 22 - - def test_find_failing(self): - a = BitArray() - p = a.find('0b1') - assert p == () - a = BitArray('0b11111111111011') - p = a.find('0b100') - assert not p - - def test_find_failing2(self): - s = BitArray('0b101') - p, = s.find('0b1', start=2) - assert p == 2 - - def test_rfind(self): - a = BitArray('0b1000000') - p = a.rfind('0b1') - assert p == (6,) - p = a.rfind('0b000') - assert p == (3,) - - def test_rfind_with_start_and_end(self): - a = BitArray('0b11 0000 11 00') - p = a.rfind('0b11', start=8) - assert p[0] == 8 - p = a.rfind('0b110', start=8) - assert p == () - p = a.rfind('0b11', end=-1) - assert p[0] == 2 - - def test_findall(self): - a = BitArray('0b001000100001') - b = list(a.findall('0b1')) - assert b == [0, 5, 9] - c = list(a.findall('0b0001')) - assert c == [0, 5] - d = list(a.findall('0b10')) - assert d == [4, 8] - e = list(a.findall('0x198273641234')) - assert e == [] - - def test_find_all_with_start_and_end(self): - a = BitArray('0xaabbccaabbccccbb') - b = list(a.findall('0xbb', start=0, end=8)) - assert b == [0] - b = list(a.findall('0xbb', start=1, end=8)) - assert b == [] - b = list(a.findall('0xbb', start=0, end=7)) - assert b == [] - b = list(a.findall('0xbb', start=48)) - assert b == [48] - b = list(a.findall('0xbb', start=47)) - assert b == [48] - b = list(a.findall('0xbb', start=49)) - assert b == [] - - def test_find_all_byte_aligned(self): - a = BitArray('0x0550550') - b = list(a.findall('0x55', bytealigned=True)) - assert b == [16] - - def test_find_all_with_count(self): - a = BitArray('0b0001111101') - b = list(a.findall([1], start=1, count=1)) - assert b == [2] - - def test_split(self): - a = BitArray('0x4700004711472222') - li = list(a.split('0x47', bytealigned=True)) - assert li == ['', '0x472222', '0x4711', '0x470000'] - - def test_byte_swap(self): - a = BitArray('0xaa00ff00ff00') - n = a.byteswap(2, end=32, repeat=True) - assert n == 2 - assert a == '0xaa0000ff00ff' - - def test_insert(self): - a = BitArray('0x0123456') - a.insert('0xf', 4) - assert a == '0x012345f6' - - def test_overwrite(self): - a = BitArray('0x00000000') - a.overwrite('0xdead', 4) - assert a == '0x000dead0' - - def test_replace(self): - a = BitArray('0x5551100') - n = a.replace('0x1', '0xabc') - assert n == 2 - assert a == '0x555abcabc00' - n = a.replace([1], [0], end=12) - assert n == 2 - assert a == '0x555abcab000' - - def test_reverse(self): - a = BitArray('0x0011223344') - a.reverse() - assert a == '0x22cc448800' - a.reverse(0, 16) - assert a == '0x22cc440011' - - def test_ror(self): - a = BitArray('0b111000') - a.ror(1) - assert a == '0b011100' - a = BitArray('0b111000') - a.ror(1, start=2, end=6) - assert a == '0b011100' - - def test_rol(self): - a = BitArray('0b1') - a.rol(12) - assert a == '0b1' - b = BitArray('0b000010') - b.rol(3) - assert b == '0b010000' - - def test_set(self): - a = BitArray(100) - a.set(1, [0, 2, 4]) - assert a[0] - assert a.startswith('0b000010101') - a = BitArray('0b111') - a.set(False, 0) - assert a == '0b110' - - def test_failing_repr(self): - a = BitArray('0b010') - a.find('0b1') - assert repr(a) == "BitArray('0b010')" - - def test_left_shift(self): - a = BitArray('0b11001') - assert (a << 1).b == '10010' - assert (a << 5).b == '00000' - assert (a << 0).b == '11001' - - def test_right_shift(self): - a = BitArray('0b11001') - assert (a >> 1).b == '01100' - assert (a >> 5).b == '00000' - assert (a >> 0).b == '11001' - - # def testConstFileBased(self): - # filename = os.path.join(THIS_DIR, 'test.m1v') - # a = Bits(filename=filename, offset=8) - # self.assertTrue(a[-8]) - # self.assertTrue(a.endswith('0x01b3')) - class TestRepr: @@ -651,20 +460,29 @@ class TestNewProperties: def test_aliases(self): a = BitArray('0x1234567890ab') - assert a.oct == a.o - assert a.hex == a.h - assert a.bin == a.b assert a[:32].float == a[:32].f assert a.int == a.i assert a.uint == a.u + def test_removed_string_aliases(self): + a = BitArray('0x123') + for alias in ['b', 'o', 'h', 'b12', 'o12', 'h12']: + with pytest.raises(AttributeError): + getattr(a, alias) + with pytest.raises(bitstring.CreationError): + setattr(a, alias, '0') + with pytest.raises(ValueError): + bitstring.Dtype(alias) + with pytest.raises(ValueError): + a.unpack(alias) + def test_aliases_with_lengths(self): a = BitArray('0x123') - h = a.h12 + h = a.hex12 assert h == '123' - b = a.b12 + b = a.bin12 assert b == '000100100011' - o = a.o12 + o = a.oct12 assert o == '0443' u = a.u12 assert u == a.u @@ -682,15 +500,15 @@ def test_assignments(self): assert a.u88 == 1244322 a.i3 = -3 assert a.i3 == -3 - a.h16 = '0x1234' - assert a.h16 == '1234' - a.o9 = '0o765' - assert a.o9 == '765' - a.b7 = '0b0001110' - assert a.b7 == '0001110' + a.hex16 = '0x1234' + assert a.hex16 == '1234' + a.oct9 = '0o765' + assert a.oct9 == '765' + a.bin7 = '0b0001110' + assert a.bin7 == '0001110' def test_assignments_without_length(self): - a = BitArray(64) + a = BitArray.from_zeros(64) a.f = 1234.5 assert a.float == 1234.5 assert a.len == 64 @@ -700,21 +518,21 @@ def test_assignments_without_length(self): a.i = -999 assert a.int == -999 assert a.len == 64 - a.h = 'feedbeef' + a.hex = 'feedbeef' assert a.hex == 'feedbeef' - a.o = '1234567' + a.oct = '1234567' assert a.oct == '1234567' - a.b = '001' + a.bin = '001' assert a.bin == '001' def test_getter_length_errors(self): a = BitArray('0x123') with pytest.raises(bitstring.InterpretError): - _ = a.h16 + _ = a.hex16 with pytest.raises(bitstring.InterpretError): - _ = a.b3317777766 + _ = a.bin3317777766 with pytest.raises(AttributeError): - _ = a.o2 + _ = a.oct2 with pytest.raises(bitstring.InterpretError): _ = a.f with pytest.raises(bitstring.InterpretError): @@ -743,10 +561,10 @@ def test_setter_length_errors(self): a.hex4 = '0xab' assert len(a) == 64 with pytest.raises(bitstring.CreationError): - a.o3 = '0xab' + a.oct3 = '0xab' with pytest.raises(bitstring.CreationError): - a.b4 = '0xab' - a.h0 = '' + a.bin4 = '0xab' + a.hex0 = '' assert a.len == 0 a.i8 = 127 a.i8 = -128 @@ -759,15 +577,16 @@ def test_setter_length_errors(self): def test_unpack(self): a = BitArray('0xff160120') - b = a.unpack('h8,2*u12') + b = a.unpack('hex8,2*u12') assert b == ['ff', 352, 288] def test_reading(self): - a = bitstring.BitStream.fromstring('0x01ff') - b = a.read('u8') + a = BitArray.from_string('0x01ff') + r = bitstring.Reader(a) + b = r.read('u8') assert b == 1 - assert a.pos == 8 - assert a.read('i') == -1 + assert r.pos == 8 + assert r.read('i') == -1 def test_longer_more_general_names(self): a = BitArray() @@ -829,7 +648,7 @@ def test_creation_errors(self): _ = BitArray('bfloat:1=0.5') def test_little_endian(self): - a = BitArray.fromstring('f32=1000') + a = BitArray.from_string('f32=1000') b = BitArray(bfloat=a.f) assert a[0:16] == b[0:16] @@ -855,15 +674,15 @@ def test_more_creation(self): assert (x, y, z) == (1.0, 2.0, 3.0) def test_interpret_bug(self): - a = BitArray(100) + a = BitArray.from_zeros(100) with pytest.raises(bitstring.InterpretError): v = a.bfloat def test_overflows(self): s = BitArray() - inf16 = BitArray(float=float('inf'), length=16) - inf32 = BitArray(float=float('inf'), length=32) - inf64 = BitArray(float=float('inf'), length=64) + inf16 = BitArray(f=float('inf'), length=16) + inf32 = BitArray(f=float('inf'), length=32) + inf64 = BitArray(f=float('inf'), length=64) infbfloat = BitArray(bfloat=float('inf')) s.f64 = 1e400 @@ -875,9 +694,9 @@ def test_overflows(self): s.bfloat = 1e60 assert s == infbfloat - ninf16 = BitArray(float=float('-inf'), length=16) - ninf32 = BitArray(float=float('-inf'), length=32) - ninf64 = BitArray(float=float('-inf'), length=64) + ninf16 = BitArray(f=float('-inf'), length=16) + ninf32 = BitArray(f=float('-inf'), length=32) + ninf64 = BitArray(f=float('-inf'), length=64) ninfbfloat = BitArray(bfloat=float('-inf')) s.f64 = -1e400 @@ -911,72 +730,6 @@ def test_native_endian_string_initialisers(self): THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -class TestBitarray: - - def teardown_method(self) -> None: - bitstring.lsb0 = False - - def test_to_bitarray(self): - a = BitArray('0xff, 0b0') - if a._bitstore.using_rust_core(): - with pytest.raises(TypeError): - _ = a.tobitarray() - else: - b = a.tobitarray() - assert type(b) == bitarray.bitarray - assert b == bitarray.bitarray('111111110') - - def test_to_bitarray_lsb0(self): - bitstring.lsb0 = True - a = bitstring.Bits('0xff, 0b0') - if a._bitstore.using_rust_core(): - with pytest.raises(TypeError): - _ = a.tobitarray() - else: - b = a.tobitarray() - assert type(b) == bitarray.bitarray - assert b == bitarray.bitarray('111111110') - - def test_from_file(self): - a = bitstring.ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile')) - if a._bitstore.using_rust_core(): - with pytest.raises(TypeError): - _ = a.tobitarray() - else: - b = a.tobitarray() - assert a.bin == b.to01() - - def test_with_offset(self): - a = bitstring.ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile')) - b = bitstring.ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile'), offset=11) - assert len(a) == len(b) + 11 - if a._bitstore.using_rust_core(): - with pytest.raises(TypeError): - _ = a.tobitarray() - else: - assert a[11:].tobitarray() == b.tobitarray() - - def test_with_length(self): - a = bitstring.ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile')) - b = bitstring.ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile'), length=11) - assert len(b) == 11 - if a._bitstore.using_rust_core(): - with pytest.raises(TypeError): - _ = a.tobitarray() - else: - assert a[:11].tobitarray() == b.tobitarray() - - def test_with_offset_and_length(self): - a = bitstring.ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile')) - b = bitstring.ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile'), offset=17, length=7) - assert len(b) == 7 - if a._bitstore.using_rust_core(): - with pytest.raises(TypeError): - _ = a.tobitarray() - else: - assert a[17:24].tobitarray() == b.tobitarray() - - try: import numpy as np numpy_installed = True @@ -1003,7 +756,9 @@ def test_setting(self): @pytest.mark.skipif(not numpy_installed, reason="numpy not installed.") def test_creation(self): - a = BitArray(np.longlong(12)) + with pytest.raises(TypeError, match="from_zeros"): + _ = BitArray(np.longlong(12)) + a = BitArray.from_zeros(np.longlong(12)) assert a.hex == '000' @@ -1023,4 +778,4 @@ def test_bytes_from_list(): def test_slice_bug(): a = BitArray('0xffff') a[0:8] = '0x1234' - assert a == '0x1234ff' \ No newline at end of file + assert a == '0x1234ff' diff --git a/tests/test_bitarray_legacy.py b/tests/test_bitarray_legacy.py new file mode 100644 index 00000000..f1fc2728 --- /dev/null +++ b/tests/test_bitarray_legacy.py @@ -0,0 +1,2914 @@ +#!/usr/bin/env python + +import collections +import copy +import os +import sys + +import pytest + +import bitstring +from bitstring import BitArray, Bits, Dtype, Reader, pack + + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ('uint:8=12', BitArray(uint=12, length=8)), + (' uint : 8 = 12', BitArray(uint=12, length=8)), + (' int:2= -1', BitArray(int=-1, length=2)), + ('int :2 = -1', BitArray(int=-1, length=2)), + (' int: 2 =-1 ', BitArray(int=-1, length=2)), + ('hex=12', '0x12'), + ('oct=33', '0o33'), + ('bin=10', '0b10'), + ('se=-1', 'se=-1'), + (' se = -1 ', 'se=-1'), + ('ue=23', 'ue=23'), + ('ue = 23', 'ue=23'), + ], +) +def test_flexible_initialisation(source, expected): + assert BitArray(source) == expected + + +def test_multiple_string_initialisation_with_reader(): + a = BitArray('0b1 , 0x1') + assert a == '0b10001' + r = Reader(BitArray('ue=5, ue=1, se=-2')) + assert r.read('ue') == 5 + assert r.read('ue') == 1 + assert r.read('se') == -2 + r = Reader(BitArray('uint:32 = 12, 0b11') + 'int:100=-100, 0o44') + assert r.read(32).uint == 12 + assert r.read(2).bin == '11' + assert r.read(100).int == -100 + + +def test_reader_reads_bits_and_bytes(): + s = Reader(BitArray(bytes=b'\x4d\x55')) + assert s.read(4).hex == '4' + assert s.read(8).hex == 'd5' + assert s.read(1) == [0] + assert s.read(3).bin == '101' + assert not s.read(0) + + s = Reader(BitArray(hex='4d55')) + assert s.read(8).hex == '4d' + assert s.read(8).hex == '55' + + s = Reader(BitArray(hex='0x112233448811')) + assert s.read(3 * 8).hex == '112233' + with pytest.raises(ValueError): + s.read(-16) + s.bitpos += 1 + assert s.read(2 * 8).bin == '1000100100010000' + + +def test_reader_reads_exp_golomb_codes(): + with pytest.raises(bitstring.InterpretError): + _ = Bits('').ue + s = Reader(BitArray(bin='1 010 011 00100 00101 00110 00111 0001000 0001001')) + assert s.pos == 0 + for i in range(9): + assert s.read('ue') == i + with pytest.raises(bitstring.ReadError): + s.read('ue') + + s = Reader(BitArray(bin='010 00110 0001010 0001000 00111')) + assert s.read('se') == 1 + assert s.read('se') == 3 + assert s.readlist(3 * ['se']) == [5, 4, -3] + + +def test_reader_find_variants(): + s = Reader(Bits(bin='0b0000110110000')) + assert s.find(BitArray(bin='11011')) == 4 + assert s.bitpos == 4 + assert s.read(5).bin == '11011' + s.bitpos = 0 + assert s.find('0b11001', bytealigned=False) is None + + s = Reader(BitArray(bin='0')) + assert s.find(s.bits, bytealigned=False) == 0 + assert s.pos == 0 + assert s.find('0b00', bytealigned=False) is None + with pytest.raises(ValueError): + s.find(BitArray()) + + s = Reader(BitArray(hex='0x112233')[4:]) + assert s.find('0x23', bytealigned=False) == 8 + assert s.pos == 8 + + +def test_reader_find_corner_cases(): + s = Reader(BitArray(bin='000111000111')) + assert s.find('0b000') == 0 + assert s.pos == 0 + assert s.find('0b000') == 0 + assert s.pos == 0 + assert s.find('0b0111000111') == 2 + assert s.pos == 2 + assert s.find('0b000', start=2) == 6 + assert s.pos == 6 + assert s.find('0b111', start=6) == 9 + assert s.pos == 9 + s.pos += 2 + assert s.find('0b1', start=s.pos) == 11 + + +def test_reader_find_byte_aligned(): + s = Reader(BitArray.from_string('0x010203040102ff')) + assert s.find('0x05', bytealigned=True) is None + assert s.find('0x02', bytealigned=True) == 8 + assert s.read(16).hex == '0203' + assert s.find('0x02', start=s.bitpos, bytealigned=True) == 40 + s.read(1) + assert s.find('0x02', start=s.bitpos, bytealigned=True) is None + + s = Reader(BitArray(hex='0x12345678')) + assert s.find(BitArray(hex='0x56'), bytealigned=True) == 16 + assert s.bytepos == 2 + s.pos = 0 + assert s.find(BitArray(hex='0x45'), bytealigned=True) is None + + +def test_reader_rfind_variants(): + a = Reader(BitArray('0b001001001')) + b = a.rfind('0b001') + assert b == 6 + assert a.pos == 6 + big = BitArray(length=100000) + '0x12' + BitArray(length=10000) + r = Reader(big) + found = r.rfind('0x12', bytealigned=True) + assert found == 100000 + assert r.pos == 100000 + + a = Reader(BitArray('0x8888')) + b = a.rfind('0b1', bytealigned=True) + assert b == 8 + assert a.pos == 8 + + a = Reader(BitArray('0x0000ffffff')) + b = a.rfind('0x0000', start=1, bytealigned=True) + assert b is None + assert a.pos == 0 + b = a.rfind('0x00', start=1, bytealigned=True) + assert b == 8 + assert a.pos == 8 + + +def test_reader_find_and_rfind_errors(): + s = Reader(BitArray(hex='0xffff')) + with pytest.raises(ValueError): + s.find('') + with pytest.raises(ValueError): + s.find(BitArray()) + a = Reader(BitArray('0x43234234')) + with pytest.raises(ValueError): + a.rfind('', bytealigned=True) + with pytest.raises(ValueError): + a.rfind('0b1', start=-99, bytealigned=True) + with pytest.raises(ValueError): + a.rfind('0b1', end=33, bytealigned=True) + with pytest.raises(ValueError): + a.rfind('0b1', start=10, end=9, bytealigned=True) + + +def test_shift_left_and_right(): + s = BitArray.from_string('0b1010') + t = s << 1 + assert s.bin == '1010' + assert t.bin == '0100' + s = t << 0 + assert s == '0b0100' + t = s << 100 + assert t.bin == '0000' + + s = BitArray('0b1010') + t = s >> 1 + assert s.bin == '1010' + assert t.bin == '0101' + q = s >> 0 + assert q == '0b1010' + q.replace('0b1010', '') + t = s >> 100 + assert t.bin == '0000' + + +def test_shift_errors(): + s = BitArray() + with pytest.raises(ValueError): + s << 1 + with pytest.raises(ValueError): + s >> 1 + s = BitArray('0xf') + with pytest.raises(ValueError): + s << -1 + with pytest.raises(ValueError): + s >> -1 + + +def test_shift_in_place(): + s = BitArray.from_string('0xffff')[4:12] + s >>= 1 + assert s == '0b01111111' + s = BitArray('0b11011') + s >>= 2 + assert s.bin == '00110' + s >>= 100000000000000 + assert s.bin == '00000' + s = BitArray('0xff') + s >>= 1 + assert s == '0x7f' + s >>= 0 + assert s == '0x7f' + + s = BitArray('0xffff')[4:12] + s <<= 2 + assert s == '0b11111100' + s = BitArray('0b11011') + s <<= 2 + assert s.bin == '01100' + s <<= 100000000000000000000 + assert s.bin == '00000' + + +def test_replace_variants(): + a = BitArray('0b1') + n = a.replace('0b1', '0b0', bytealigned=True) + assert a.bin == '0' + assert n == 1 + assert a.replace('0b1', '0b0', bytealigned=True) == 0 + + a = BitArray('0b00001111111') + n = a.replace('0b1', '0b0', bytealigned=True) + assert a.bin == '00001111011' + assert n == 1 + n = a.replace('0b1', '0b0', bytealigned=False) + assert a.bin == '00000000000' + assert n == 6 + + a = BitArray('0b0') + n = a.replace('0b0', '0b110011111', bytealigned=True) + assert n == 1 + assert a.bin == '110011111' + n = a.replace('0b11', '', bytealigned=False) + assert n == 3 + assert a.bin == '001' + + +def test_replace_count_and_self(): + a = BitArray('0b11') + a.replace('0b1', a) + assert a == '0xf' + a.replace(a, a) + assert a == '0xf' + + a = BitArray('0x223344223344223344') + n = a.replace('0x2', '0x0', count=0, bytealigned=True) + assert n == 0 + assert a.hex == '223344223344223344' + n = a.replace('0x2', '0x0', count=1, bytealigned=True) + assert n == 1 + assert a.hex == '023344223344223344' + n = a.replace('0x33', '', count=2, bytealigned=True) + assert n == 2 + assert a.hex == '02442244223344' + n = a.replace('0x44', '0x4444', count=1435, bytealigned=True) + assert n == 3 + assert a.hex == '02444422444422334444' + + +def test_replace_does_not_adjust_reader_position(): + a = BitArray('0xff') + r = Reader(a, pos=8) + a.replace('0xff', '', bytealigned=True) + assert a == '' + assert r.pos == 8 + + +def test_replace_errors(): + a = BitArray('0o123415') + with pytest.raises(ValueError): + a.replace('', Bits.from_zeros(0o7), bytealigned=True) + with pytest.raises(ValueError): + a.replace('0b1', '0b1', start=-100, bytealigned=True) + with pytest.raises(ValueError): + a.replace('0b1', '0b1', end=19, bytealigned=True) + + +def test_slice_assignment(): + a = BitArray() + a[0:0] = '0xabcdef' + a[4:16] = '' + assert a == '0xaef' + a[8:] = '0x00' + assert a == '0xae00' + a += '0xf' + a[8:] = '0xe' + assert a == '0xaee' + b = BitArray("0x0000") + b[0:16] = '0xffee' + assert b == '0xffee' + b[4:16] = '0xeed123' + assert b == '0xfeed123' + b[0:8] = '0x0000' + assert b == '0x0000ed123' + + +def test_inserting_using_set_item(): + a = BitArray() + a[0:0] = '0xdeadbeef' + assert a == '0xdeadbeef' + a[16:16] = '0xfeed' + assert a == '0xdeadfeedbeef' + a[0:0] = '0xa' + assert a == '0xadeadfeedbeef' + a[0:8] = '0xff' + a[8:0] = '0x000' + assert a.startswith('0xff000ead') + + +def test_pack_legacy_cases(): + s = bitstring.pack('uint:6, bin, hex, int:6, se, ue, oct', 10, '0b110', 'ff', -1, -6, 6, '54') + t = BitArray('uint:6=10, 0b110, 0xff, int:6=-1, se=-6, ue=6, oct=54') + assert s == t + assert type(s) is Bits + with pytest.raises(ValueError): + pack('tomato', '0') + with pytest.raises(ValueError): + pack('uint', 12) + with pytest.raises(ValueError): + pack('int', 12) + with pytest.raises(ValueError): + pack('hex', 'penguin') + with pytest.raises(ValueError): + pack('hex12', '0x12') + + +@pytest.mark.parametrize( + ("fmt", "expected"), + [ + ('0xf', '0xf'), + ('0b1', '0b1'), + ('0o7', '0o7'), + ('int:10=-1', '0b1111111111'), + ('uint:10=1', '0b0000000001'), + ('bin=01', '0b01'), + ('hex=01', '0x01'), + ('oct=01', '0o01'), + ], +) +def test_pack_literals(fmt, expected): + assert pack(fmt) == expected + + +def test_pack_with_dicts_and_lists(): + a = pack('uint:6=width, se=height', height=100, width=12) + w, h = a.unpack('uint:6, se') + assert w == 12 + assert h == 100 + d = {'w': '0xf', '300': 423, 'e': '0b1101'} + a = pack('int:100=300, bin=e, uint:12=300', **d) + x, y, z = a.unpack('int:100, bin, uint:12') + assert x == 423 + assert y == '1101' + assert z == 423 + + a = pack('int:5, bin:3=b, 0x3, bin=c, se=12', 10, b='0b111', c='0b1') + b = BitArray('int:5=10, 0b111, 0x3, 0b1, se=12') + assert a == b + assert pack('bits:3=b', b=BitArray('0b101')) == '0b101' + assert pack('bits:24=b', b=BitArray('0x001122')) == '0x001122' + + f = ['bin', 'hex', 'uint:10'] + a = pack(','.join(f), '00', '234', 100) + b = pack(f, '00', '234', 100) + assert a == b + + +def test_pack_length_restrictions_and_null_formats(): + _ = pack('bin:3', '0b000') + with pytest.raises(bitstring.CreationError): + _ = pack('bin:3', '0b0011') + with pytest.raises(bitstring.CreationError): + _ = pack('bin:3', '0b11') + _ = pack('hex:4', '0xf') + with pytest.raises(bitstring.CreationError): + _ = pack('hex:4', '0b111') + _ = pack('oct:6', '0o77') + with pytest.raises(bitstring.CreationError): + _ = pack('oct:6', '0o1') + _ = pack('bits:3', BitArray('0b111')) + with pytest.raises(bitstring.CreationError): + _ = pack('bits:3', BitArray('0b11')) + + assert not pack('') + assert not pack(',') + assert pack(',,,,,0b1,,,,,,,,,,,,,0b1,,,,,,,,,,') == '0b11' + s = pack(',,uint:12,,bin:3,', 100, '100') + a, b = s.unpack('uint:12,bin:3') + assert a == 100 + assert b == '100' + + +def test_unpack_does_not_depend_on_reader_position(): + bits = BitArray('uint:13=23, hex=e, bin=010, int:41=-554, 0o44332, se=-12, ue=4') + r = Reader(bits, pos=11) + a, b, c, d, e, f, g = bits.unpack('uint:13, hex:4, bin:3, int:41, oct:15, se, ue') + assert (a, b, c, d, e, f, g) == (23, 'e', '010', -554, '44332', -12, 4) + assert r.pos == 11 + + +def test_unpack_stretchy_tokens(): + s = BitArray('0xff, 0b000, uint:12=100') + a, b, c = s.unpack('bits:8, bits, uint:12') + assert a == '0xff' + assert b == '0b000' + assert c == 100 + a, b = s.unpack(['bits:11', 'uint']) + assert a == '0xff, 0b000' + assert b == 100 + + +def test_file_creation_and_mutation_operations(): + filename = os.path.join(THIS_DIR, 'smalltestfile') + s = BitArray(filename=filename) + s.append('0xff') + assert s.hex == '0123456789abcdefff' + + s = Bits(filename=filename) + t = BitArray('0xff') + s + assert t.hex == 'ff0123456789abcdef' + + s = BitArray(filename=filename) + del s[:1] + assert (BitArray('0b0') + s).hex == '0123456789abcdef' + + s = BitArray(filename=filename) + del s[:7 * 8] + assert s.hex == 'ef' + + s = BitArray(filename=filename) + s.insert('0xc', 4) + assert s.hex == '0c123456789abcdef' + + s = BitArray(filename=filename) + s.prepend('0xf') + assert s.hex == 'f0123456789abcdef' + + s = BitArray(filename=filename) + s.overwrite('0xaaa', 12) + assert s.hex == '012aaa6789abcdef' + + s = BitArray(filename=filename) + s.reverse() + assert s.hex == 'f7b3d591e6a2c480' + + +def test_file_properties_and_reader(): + s = Bits(filename=os.path.join(THIS_DIR, 'smalltestfile')) + assert s.hex == '0123456789abcdef' + assert s.uint == 81985529216486895 + assert s.int == 81985529216486895 + assert s.bin == '0000000100100011010001010110011110001001101010111100110111101111' + assert s[:-1].oct == '002215053170465363367' + r = Reader(s) + assert r.read('se') == -72 + r.pos = 0 + assert r.read('ue') == 144 + assert s.bytes == b'\x01\x23\x45\x67\x89\xab\xcd\xef' + assert s.tobytes() == b'\x01\x23\x45\x67\x89\xab\xcd\xef' + + +def test_file_creation_with_length_and_offset(): + test_filename = os.path.join(THIS_DIR, 'test.m1v') + s = Bits(filename=test_filename, length=32) + assert s.length == 32 + assert s.hex == '000001b3' + assert s.bytes == b'\x00\x00\x01\xb3' + assert s.uint == 0x1b3 + assert s.int == 0x1b3 + assert s.bin == '00000000000000000000000110110011' + assert not Bits(filename=test_filename, length=0) + + small_test_filename = os.path.join(THIS_DIR, 'smalltestfile') + with pytest.raises(bitstring.CreationError): + _ = BitArray(filename=small_test_filename, length=65) + with pytest.raises(bitstring.CreationError): + _ = Bits(filename=small_test_filename, length=64, offset=1) + with pytest.raises(bitstring.CreationError): + _ = Bits(filename=small_test_filename, offset=65) + + a = BitArray(filename=test_filename, offset=4) + assert Reader(a).peek(4 * 8).hex == '00001b31' + b = BitArray(filename=test_filename, offset=28) + assert Reader(b).peek(8).hex == '31' + + +def test_find_in_file_with_reader(): + r = Reader(BitArray(filename=os.path.join(THIS_DIR, 'test.m1v'))) + assert r.find('0x160120') == 32 + assert r.bytepos == 4 + s3 = r.read(24) + assert s3.hex == '160120' + r.bytepos = 0 + assert r.pos == 0 + assert r.find('0x0001b2') == 104 + assert r.bytepos == 13 + + +def test_file_bit_getting(): + s = Bits(filename=os.path.join(THIS_DIR, 'smalltestfile'), offset=16, length=8) + assert s[1] + assert s.any(0, [-1, -2, -3]) + assert not s.all(0, [0, 1, 2]) + + +def test_creation_errors_and_basic_conversions(): + s = BitArray() + with pytest.raises(bitstring.CreationError): + s._setbin('0010020') + with pytest.raises(bitstring.CreationError): + s.hex = '0xabcdefg' + assert BitArray('').len == 0 + assert BitArray('0x80').len == 8 + with pytest.raises(bitstring.CreationError): + BitArray(hex='0xffff', offset=-1) + assert BitArray('0x10').uint == 16 + assert BitArray('0b000111').uint == 7 + assert BitArray('0x10').int == 16 + assert BitArray('0b11110').int == -2 + assert BitArray(bytes=b'\x00\x12\x23\xff').hex == '001223ff' + with pytest.raises(bitstring.InterpretError): + _ = BitArray('0b11111').hex + + +def test_empty_bits_and_reader_position(): + s = Reader(BitArray()) + with pytest.raises(bitstring.ReadError): + s.read(1) + assert s.bits.bin == '' + assert s.bits.hex == '' + with pytest.raises(bitstring.InterpretError): + _ = s.bits.int + with pytest.raises(bitstring.InterpretError): + _ = s.bits.uint + assert not s.bits + + s = Reader(BitArray(bytes=b'\x00\x00\x00')) + assert s.bitpos == 0 + s.read(5) + assert s.pos == 5 + s.pos = len(s) + with pytest.raises(bitstring.ReadError): + s.read(1) + + +def test_reader_byte_position_and_lax_seek(): + s = Reader(BitArray(bytes=b'\x00\x00\x00')) + assert s.bytepos == 0 + s.read(10) + with pytest.raises(bitstring.ByteAlignError): + _ = s.bytepos + s.read(6) + assert s.bytepos == 2 + + s = Reader(BitArray(bytes=b'\x00\x00\x00\x00\x00\xab')) + s.bytepos = 5 + assert s.read(8).hex == 'ab' + s.pos = -1 + assert s.pos == -1 + with pytest.raises(ValueError): + s.read(1) + s.bitpos = 6 * 8 + 1 + assert s.bitpos == 49 + with pytest.raises(ValueError): + s.read(0) + + +def test_append_and_prepend(): + s1 = BitArray('0b00000') + s1.append(BitArray(bool=True)) + assert s1.bin == '000001' + assert (BitArray('0x0102') + BitArray('0x0304')).hex == '01020304' + + s1 = BitArray('0xf0')[:6] + s1.append(s1) + assert s1.bin == '111100111100' + + s = BitArray(bytes=b'\x28\x28', offset=1) + s.append('0b0') + assert s.hex == '5050' + + s = BitArray('0b000') + s.prepend('0b11') + assert s.bin == '11000' + s.prepend(s) + assert s.bin == '1100011000' + s.prepend('') + assert s.bin == '1100011000' + + +def test_reader_bytealign(): + s = Reader(BitArray(hex='0001ff23')) + s.bytealign() + assert s.bytepos == 0 + s.pos += 11 + s.bytealign() + assert s.bytepos == 2 + s.pos -= 10 + s.bytealign() + assert s.bytepos == 1 + + +def test_insert_and_overwrite(): + s = BitArray('0x0011') + s.insert(BitArray('0x22'), 8) + assert s.hex == '002211' + s = BitArray.from_zeros(0) + s.insert(BitArray(bin='101'), 0) + assert s.bin == '101' + + s1 = BitArray(hex='0x123456') + s2 = BitArray(hex='0xff') + s1.insert(s2, 8) + assert s1.hex == '12ff3456' + s1.insert('0xee', 24) + assert s1.hex == '12ff34ee56' + with pytest.raises(ValueError): + s1.insert('0b1', -1000) + with pytest.raises(ValueError): + s1.insert('0b1', 1000) + + s = BitArray(bin='0') + s.overwrite(BitArray(bin='1'), 0) + assert s.bin == '1' + s = BitArray(bin='0b11111') + s.overwrite(BitArray(bin='000'), 0) + assert s.bin == '00011' + s.overwrite('0b000', 2) + assert s.bin == '00000' + + +def test_truncate_and_delete(): + s = BitArray('0b1') + del s[0] + assert not s + s = BitArray(hex='1234') + del s[:4] + assert s.hex == '234' + del s[:9] + assert s.bin == '100' + del s[:2] + assert s.bin == '0' + del s[:1] + assert not s + + s = BitArray(bytes=b'\x12\x34') + del s[-4:] + assert s.hex == '123' + del s[-9:] + assert s.bin == '000' + del s[-3:] + assert not s + + +def test_slice_and_join(): + s = BitArray(hex='0x123456') + assert s[8:16].hex == '34' + s = s[8:24] + assert s.len == 16 + assert s.hex == '3456' + s = s[0:8] + assert s.hex == '34' + + s1 = BitArray(bin='0') + s2 = BitArray(bin='1') + s3 = BitArray(bin='000') + s4 = BitArray(bin='111') + strings = [s1, s2, s1, s3, s4] + assert BitArray().join(strings).bin == '010000111' + + +def test_split_variants(): + s = BitArray(hex='0x1234aa1234bbcc1234ffff') + delimiter = BitArray(hex='1234') + bsl = s.split(delimiter) + assert [b.hex for b in bsl] == ['', '1234aa', '1234bbcc', '1234ffff'] + + s = BitArray(hex='aa471234fedc43 47112233 47 4723 472314') + delimiter = BitArray(hex='47') + bsl = s.split(delimiter, start=0) + assert [b.hex for b in bsl] == ['aa', '471234fedc43', '47112233', '47', '4723', '472314'] + + s = BitArray(hex='aaffaaffaaffaaffaaff') + bsl = s.split(BitArray(hex='aaffaa')) + assert [b.hex for b in bsl] == ['', 'aaffaaff', 'aaffaaffaaff'] + + +def test_adding_and_equality(): + s1 = BitArray(hex='0x0102') + s2 = BitArray(hex='0x0304') + s3 = s1 + s2 + assert s1.hex == '0102' + assert s2.hex == '0304' + assert s3.hex == '01020304' + s3 += s1 + assert s3.hex == '010203040102' + assert s2[9:16].bin == '0000100' + assert s1[0:9].bin == '000000010' + s4 = BitArray(bin='000000010') + BitArray(bin='0000100') + assert s4.bin == '0000000100000100' + s5 = s1[0:9] + s2[9:16] + assert s5.bin == '0000000100000100' + + s1 = BitArray('0b01010101') + s2 = BitArray('0b01010101') + assert s1 == s2 + assert BitArray(bytes=b'\xff', offset=2, length=3) == BitArray('0b111') + + +def test_peek_with_reader(): + s = Reader(BitArray(bin='01')) + assert s.peek(1) == [0] + assert s.peek(1) == [0] + assert s.read(1) == [0] + assert s.peek(1) == [1] + assert s.peek(1) == [1] + + s = Reader(BitArray(bytes=b'\x1f', offset=3)) + assert len(s) == 5 + assert s.peek(5).bin == '11111' + s.pos += 1 + with pytest.raises(bitstring.ReadError): + _ = s.peek(5) + + s = Reader(BitArray(hex='001122334455')) + assert s.peek(8).hex == '00' + assert s.read(8).hex == '00' + s.pos += 33 + with pytest.raises(bitstring.ReadError): + _ = s.peek(8) + + +def test_auto_initialisation_and_auto_methods(): + assert BitArray('0xff').hex == 'ff' + assert BitArray('0b00011').bin == '00011' + with pytest.raises(bitstring.CreationError): + _ = BitArray('hello') + with pytest.raises(TypeError): + _ = BitArray(1.2) + + s = BitArray('0xff') + s.insert('0x00', 4) + assert s.hex == 'f00f' + with pytest.raises(ValueError): + s.insert('ff', 0) + + s = BitArray('0x0110') + s.overwrite('0b1', 0) + assert s.hex == '8110' + s.overwrite('', 0) + assert s.hex == '8110' + with pytest.raises(ValueError): + s.overwrite('0bf', 0) + + +def test_findall_contains_and_slice_steps(): + a = BitArray('0b11111') + assert list(a.findall('0b1')) == [0, 1, 2, 3, 4] + assert list(a.findall('0b11')) == [0, 1, 2, 3] + assert list(a.findall('0b10')) == [] + a = BitArray('0x4733eeff66554747335832434547') + assert list(a.findall('0x47', bytealigned=True)) == [0, 6 * 8, 7 * 8, 13 * 8] + assert list(a.findall('0x4733', bytealigned=True)) == [0, 7 * 8] + + a = BitArray('0b1') + '0x0001dead0001' + assert '0xdead' in a + assert '0xfeed' not in a + + a = BitArray('0b0011000111') + assert a[7:3:-1] == '0b1000' + assert a[9:2:-1] == '0b1110001' + assert a[8:2:-2] == '0b100' + assert a[100:-20:-3] == '0b1010' + + +def test_repr_print_iter_and_offsets(): + for bs in ['', '0b1', '0o5', '0x43412424f41', '0b00101001010101']: + a = BitArray(bs) + b = eval(a.__repr__()) + assert a == b + assert repr(BitArray('0b1')) == "BitArray('0b1')" + assert str(BitArray('0b11010')) == '0b11010' + + a = BitArray('0b001010') + b = BitArray() + for bit in a: + b.append(Bits(bool=bit)) + assert a == b + + a = BitArray(bytes=b'\xff', offset=2) + b = BitArray('0b00') + b += a + assert b == '0b0011 1111' + assert a.tobytes() == b'\xfc' + + +def test_init_slice_with_int_and_reverse(): + a = BitArray(length=8) + a[:] = 100 + assert a.uint == 100 + a[0] = 1 + assert a.bin == '11100100' + a[1] = 0 + assert a.bin == '10100100' + a[-1] = -1 + assert a.bin == '10100101' + a[-3:] = -2 + assert a.bin == '10100110' + + with pytest.raises(ValueError): + a[0:4] = 16 + with pytest.raises(ValueError): + a[0:4] = -9 + + a = BitArray('0x0012ff') + a.reverse() + assert a == '0xff4800' + a.reverse(start=8, end=16) + assert a == '0xff1200' + with pytest.raises(ValueError): + a.reverse(start=-1, end=4) + + +def test_initialise_from_iterables_and_cut(): + a = BitArray.from_bools([]) + assert not a + a = BitArray.from_bools([True, False, [], [0], 'hello']) + assert a == '0b10011' + a += [] + assert a == '0b10011' + a += [True, False, True] + assert a == '0b10011101' + assert [1, 0, False, True] == BitArray('0b1001') + assert [True] + BitArray('0b1') == '0b11' + + a = BitArray.from_bools(()) + assert not a + a = BitArray.from_bools((0, 1, '0', '1')) + assert '0b0111' == a + a.replace((True, True), []) + assert a == (False, True) + + a = BitArray('0x00112233445') + assert list(a.cut(8)) == ['0x00', '0x11', '0x22', '0x33', '0x44', '0x5'] + assert list(a.cut(4, start=8, end=16)) == ['0x1', '0x1'] + assert list(a.cut(4, start=0, end=44, count=4)) == ['0x0', '0x0', '0x1', '0x1'] + + +def test_reader_intelligent_reads(): + a = Reader(BitArray(uint=123, length=23)) + assert a.read('uint:23') == 123 + assert a.pos == len(a) + b = Reader(BitArray(int=-12, length=44)) + assert b.read('int:44') == -12 + assert b.pos == len(b) + r = Reader(a.bits + b.bits) + assert r.readlist('uint:23, int:44') == [123, -12] + + a = Reader(BitArray(ue=822)) + assert a.read('ue') == 822 + b = Reader(BitArray(se=-1001)) + assert b.read('se') == -1001 + r = Reader(b.bits + 2 * a.bits) + assert r.readlist('se, ue, ue') == [-1001, 822, 822] + + a = Reader(BitArray('0x123') + '0b11101') + assert a.read('hex:12') == '123' + assert a.read(' bin : 5 ') == '11101' + + +def test_reader_intelligent_read_errors_and_peek(): + a = Reader(BitArray('0x1234')) + assert a.readlist('bin:0, oct:0, hex:0, bits:0')[:3] == ['', '', ''] + a.pos = 0 + with pytest.raises(ValueError): + _ = a.read('int:0') + with pytest.raises(ValueError): + _ = a.read('uint:0') + assert a.pos == 0 + + a = Reader(BitArray('0x123456')) + for t in ['hex:1', 'oct:1', '-5', 'fred', 'bin:-2', 'uint:p', 'uint:-2', 'int:u', 'int:-3', 'ses', 'uee', '-14']: + with pytest.raises(ValueError): + _ = a.read(t) + + a = Reader(BitArray('0b01, 0x43, 0o4, uint:23=2, se=5, ue=3')) + b, c, e = a.peeklist('bin:2, hex:8, oct:3') + assert (b, c, e) == ('01', '43', '4') + assert a.pos == 0 + a.pos = 13 + f, g, h = a.peeklist('uint:23, se, ue') + assert (f, g, h) == (2, 5, 3) + assert a.pos == 13 + + +def test_reader_multiple_bit_reads(): + s = Reader(BitArray('0x123456789abcdef')) + a, b = s.readlist([4, 4]) + assert a == '0x1' + assert b == '0x2' + c, d, e = s.readlist([8, 16, 8]) + assert c == '0x34' + assert d == '0x5678' + assert e == '0x9a' + + s = Reader(BitArray('0b1101, 0o721, 0x2234567')) + a, b, c, d = s.peeklist([2, 1, 1, 9]) + assert a == '0b11' + assert bool(b) is True + assert bool(c) is True + assert d == '0o721' + assert s.pos == 0 + + +def test_to_bytes_to_file_and_token_parser(tmp_path): + a = BitArray(bytes=b'\xab\x00') + b = a.tobytes() + assert a.bytes == b + for _ in range(7): + del a[-1:] + assert a.tobytes() == b'\xab\x00' + del a[-1:] + assert a.tobytes() == b'\xab' + + filename = tmp_path / 'temp_bitstring_unit_testing_file' + a = BitArray('0x0000ff')[:17] + with open(filename, 'wb') as f: + a.tofile(f) + b = BitArray(filename=filename) + assert b == '0x000080' + + tp = bitstring.utils.tokenparser + assert tp('hex') == (True, [('hex', None, None)]) + assert tp('hex=14') == (True, [('hex', None, '14')]) + assert tp('0xef') == (False, [('0x', None, 'ef')]) + assert tp('uint:12') == (False, [('uint', 12, None)]) + assert tp('2*bits:6') == (False, [('bits', 6, None), ('bits', 6, None)]) + + +def test_endian_synonyms_and_struct_tokens(): + s = BitArray('0x12318276ef') + assert s.int == s.ibe + assert s.uint == s.ube + assert BitArray(ibe=-100, length=16) == 'int:16=-100' + assert BitArray(ube=13, length=24) == 'int:24=13' + r = Reader(BitArray('ibe:8=2')) + assert r.read('ibe8') == 2 + r.pos = 0 + assert r.read('ube8') == 2 + + s = BitArray(uint=100, length=16) + assert s.ule == 25600 + s = BitArray(ule=100, length=16) + assert s.uint == 25600 + assert s.ule == 100 + s.ule += 5 + assert s.ule == 105 + s = pack('ule:24', 1001) + assert s.ule == 1001 + assert Reader(s).read('ule24') == 1001 + + assert pack('h', 23) == BitArray('ibe:16=23') + assert pack('>I', 23) == BitArray('ube:32=23') + with pytest.raises(bitstring.CreationError): + _ = pack('2i', 40, 40) + + s = pack('>hhl', 1, 2, 3) + assert s.unpack('>hhl') == [1, 2, 3] + s = pack('Q \tL', 1001, 43, 21, 9999) + assert s.unpack('QL') == [1001, 43, 21, 9999] + + +def test_byteswap_variants(): + a = BitArray('0x123456') + a.byteswap() + assert a == '0x563412' + b = a + '0b1' + b.byteswap() + assert '0x123456, 0b1' == b + a = BitArray('0x54') + a.byteswap() + assert a == '0x54' + a = BitArray() + a.byteswap() + assert not a + + a = BitArray('0x00112233') + a.byteswap(0, 0, 16) + assert a == '0x11002233' + a.byteswap(0, 4, 28) + assert a == '0x12302103' + a.byteswap(start=0, end=18) + assert a == '0x30122103' + with pytest.raises(ValueError): + a.byteswap(0, 10, 2) + + +def test_startswith_endswith_and_hashing(): + a = BitArray() + assert a.startswith(BitArray()) + assert not a.startswith('0b0') + a = BitArray('0x12ff') + assert a.startswith('0x1') + assert a.startswith('0b0001001') + assert not a.startswith('0x2') + + a = BitArray('0xf2341') + assert a.endswith('0x41') + assert a.endswith('0b001') + assert not a.endswith('0o34') + + with pytest.raises(TypeError): + _ = {a} + with pytest.raises(TypeError): + _ = hash([a]) + + +def test_bits_immutability_and_hashability(): + a = Bits('0x012345') + assert a == '0x012345' + b = BitArray('0xf') + a + assert b == '0xf012345' + with pytest.raises(AttributeError): + a.append(b) + with pytest.raises(AttributeError): + a.prepend(b) + with pytest.raises(TypeError): + a[0] = '0b1' + with pytest.raises(TypeError): + del a[5] + with pytest.raises(AttributeError): + a.replace('0b1', '0b0') + + s = {Bits(uint=i, length=7) for i in range(15)} + assert len(s) == 15 + s.add(Bits('0b0000011')) + assert len(s) == 15 + with pytest.raises(TypeError): + s.add(BitArray('0b0000011')) + + +def test_set_invert_and_logical_inplace(): + a = BitArray(length=16) + a.set(True, 0) + assert a == '0b10000000 00000000' + a.set(1, 15) + assert a == '0b10000000 00000001' + b = a[4:12] + b.set(True, 1) + assert b == '0b01000000' + b.set(True, -1) + assert b == '0b01000001' + with pytest.raises(IndexError): + b.set(True, -9) + + a = BitArray('0b111000') + a.invert(range(a.len)) + assert a == '0b000111' + a.invert([0, 1, -1]) + assert a == '0b110110' + a.invert() + assert a == '0b001001' + + a = BitArray('0b1101001') + a |= '0b1110000' + assert a == '0b1111001' + a &= '0b1111111' + assert a == '0b1111001' + a ^= '0b1111111' + assert a == '0b0000110' + + +def test_all_any_and_count(): + a = BitArray('0b0111') + assert a.all(True, (1, 3)) + assert not a.all(True, (0, 1, 2)) + assert a.all(True, [-1]) + assert not a.all(True, [0]) + + filename = os.path.join(THIS_DIR, 'test.m1v') + a = BitArray(filename=filename) + assert a.all(True, [31]) + a = BitArray(filename=filename) + assert a.any(True, (31, 12)) + assert a.any(False, (0, 1, 2, 3, 4)) + + assert Bits('0xf0f').count(True) == 8 + assert Bits('0xf0f').count(False) == 4 + assert BitArray().count(True) == 0 + assert BitArray().count(False) == 0 + + +def test_float_initialisation_packing_and_reading(): + for f in (0.000001, -1.0, 1.0, 0.2, -3.14159265): + assert BitArray(float=f, length=64).float == f + assert BitArray('float:64=%s' % str(f)).float == f + assert BitArray('fbe:64=%s' % str(f)).fbe == f + assert BitArray('fle:64=%s' % str(f)).fle == f + assert BitArray(float=f, length=32).float / f == pytest.approx(1.0) + assert BitArray(float=f, length=16).float == pytest.approx(f, abs=0.01) + + a = BitArray(pack('>d', 0.01)) + assert a.float == 0.01 + assert a.fbe == 0.01 + a.byteswap() + assert a.fle == 0.01 + d = pack('>5d', 10.0, 5.0, 2.5, 1.25, 0.1) + assert d.unpack('>5d') == [10.0, 5.0, 2.5, 1.25, 0.1] + + a = Reader(BitArray('fle:64=12, fbe:64=-0.01, fne:64=3e33')) + assert a.readlist('fle:64, fbe:64, fne:64') == [12.0, -0.01, 3e33] + + +def test_float_errors_and_rotations(): + a = BitArray('0x3') + with pytest.raises(bitstring.InterpretError): + _ = a.float + with pytest.raises(bitstring.CreationError): + a.float = -0.2 + for le in (8, 10, 12, 18, 30, 128, 200): + with pytest.raises(ValueError): + _ = BitArray(float=1.0, length=le) + + a = BitArray('0b11001') + a.ror(0) + assert a == '0b11001' + a.ror(1) + assert a == '0b11100' + a.ror(101) + assert a == '0b01110' + a = BitArray('0b11001') + a.rol(1) + assert a == '0b10011' + a.rol(101) + assert a == '0b00111' + + +def test_bytes_token_and_dedicated_reader_types(): + a = Reader(BitArray('0x510203')) + b = a.read('bytes:1') + assert isinstance(b, bytes) + assert b == b'\x51' + x, y, z = a.bits.unpack('uint:4, bytes:2, uint') + assert x == 5 + assert y == b'\x10\x20' + assert z == 3 + assert pack('bytes:4', b'abcd').bytes == b'abcd' + + a = Reader(BitArray('0b11, uint:43=98798798172, 0b11111'), pos=2) + assert a.read(Dtype('int43')) == 98798798172 + assert a.pos == 45 + a = Reader(BitArray('0b111, ule:40=123516, 0b111'), pos=3) + assert a.read('ule:40') == 123516 + a = Reader(BitArray('0b111, bfloat:16=-5.25, 0xffffffff'), pos=3) + assert a.read('bfloatbe') == -5.25 + + +def test_multiplicative_factors_and_brackets(): + assert BitArray('1*0b1') == '0b1' + assert BitArray('4*0xc') == '0xcccc' + assert BitArray('0b1, 0*0b0') == '0b1' + assert BitArray('0*0b1001010') == '' + + s = Reader(BitArray('0xc') * 5) + assert s.readlist('5*uint:4') == [12, 12, 12, 12, 12] + s = Bits('2*0b101, 4*uint:7=3') + assert s.unpack('2*bin:3, 3*uint:7') == ['101', '101', 3, 3, 3] + + be = bitstring.utils.expand_brackets + assert be('2*(a,b)') == 'a,b,a,b' + assert be('2*(a),3*(b)') == 'a,a,b,b,b' + with pytest.raises(ValueError): + _ = be('2*(x,y()') + + +def test_bool_token_and_integer_reads(): + a = Reader(Bits('0b1')) + assert a.bits.bool is True + assert a.read('bool') is True + assert a.bits.unpack('bool')[0] is True + b = Reader(Bits('0b0')) + assert b.bits.bool is False + assert b.peek('bool') is False + assert b.bits.unpack('bool')[0] is False + + c = pack('4*bool', False, True, 'False', 'True') + assert c == '0b0101' + with pytest.raises(bitstring.CreationError): + pack('bool', 'hello') + + a = Reader(Bits('0xffeedd')) + b = a.read(8) + assert b.hex == 'ff' + assert a.pos == 8 + b = a.peek(8) + assert b.hex == 'ee' + assert a.pos == 8 + b = a.read(1) + assert b == '0b1' + + +def test_bytes_initialisation_and_legacy_bugs(): + a = Bits(b'uint:5=2') + b = Bits(b'') + c = Bits(bytes=b'uint:5=2') + assert a.bytes == b'uint:5=2' + assert not b + assert c == b'uint:5=2' + + a = Bits(bytearray(b'uint:5=2')) + b = Bits(bytearray(4)) + c = Bits(bytes=bytearray(b'uint:5=2')) + assert a.bytes == b'uint:5=2' + assert b == '0x00000000' + assert c.bytes == b'uint:5=2' + + s = Reader(Bits('0b000000001')) + with pytest.raises(bitstring.ReadError): + _ = s.read('ue') + + s = BitArray('0x00112233') + li = list(s.split('0x22', start=8, bytealigned=True)) + assert li == ['0x11', '0x2233'] + s.replace('0x22', '0xffff', start=8, bytealigned=True) + assert s == '0x0011ffff33' + + +def test_function_negative_indices(): + s = BitArray('0b0111') + s.insert('0b0', -1) + assert s == '0b01101' + with pytest.raises(ValueError): + s.insert('0b0', -1000) + + s.reverse(start=-2) + assert s == '0b01110' + t = BitArray('0x778899abcdef') + t.reverse(start=-12, end=-4) + assert t == '0x778899abc7bf' + t.byteswap(0, -40, -16) + assert t == '0x77ab9988c7bf' + t.overwrite('0x666', -20) + assert t == '0x77ab998666bf' + + r = Reader(t) + assert r.find('0x998', bytealigned=True, start=-31) is None + assert r.find('0x998', bytealigned=True, start=-32) == 16 + assert r.pos == 16 + + +def test_pack_code_dicts_unicode_and_dict_reads(): + assert sorted(bitstring.utils.REPLACEMENTS_BE.keys()) == sorted(bitstring.utils.REPLACEMENTS_LE.keys()) + assert sorted(bitstring.utils.REPLACEMENTS_BE.keys()) == sorted(bitstring.utils.PACK_CODE_SIZE.keys()) + for key in bitstring.utils.PACK_CODE_SIZE: + be = pack(bitstring.utils.REPLACEMENTS_BE[key], 0) + le = pack(bitstring.utils.REPLACEMENTS_LE[key], 0) + assert be.len == bitstring.utils.PACK_CODE_SIZE[key] * 8 + assert le.len == be.len + + a = Bits('uint:12=34') + assert a.uint == 34 + a += '0xfe' + assert a[12:] == '0xfe' + + a = Bits('2*int:13=100, 0b111') + x, y, z = a.unpack('13, int:m, bin:q', m=13, q=3) + assert x == 'uint:13=100' + assert y == 100 + assert z == '111' + + s = Reader(BitArray('0x0102')) + x, y = s.readlist('bits8, hex:b', b=4) + assert (x, y) == ('0x01', '0') + assert s.pos == 12 + + +def test_unpack_error_and_add_empty_bits_issue(): + format_with_commas = ',bytes:2,,bytes:1,' + dp = BitArray(hex='010203').unpack(fmt=format_with_commas) + assert dp == [b'\x01\x02', b'\x03'] + + x = BitArray() + y = x + Bits('0xff') + assert y == '0xff' + z = x + bitstring.BitArray('0xff') + assert z == '0xff' + + xx = Bits() + yy = xx + Bits('0xff') + zz = xx + bitstring.BitArray('0xff') + assert yy == zz == '0xff' + + +def test_copy_and_reader_copy_semantics_for_restored_stream_coverage(): + bits = Bits('0xabc') + reader = Reader(bits, pos=11) + copied_reader = copy.copy(reader) + copied_reader.pos = 4 + assert reader.bits is copied_reader.bits + assert reader.pos == 11 + assert copied_reader.pos == 4 + + mutable = BitArray('0xabc') + copied_mutable = copy.copy(mutable) + mutable.append('0xf') + assert copied_mutable == '0xabc' + assert mutable == '0xabcf' + + +def test_iterable_detection_used_by_initialisation(): + assert isinstance(range(10), collections.abc.Iterable) + s = Bits.from_bools(range(12)) + assert s == '0x7ff' + + +def test_multiplication_variants(): + a = BitArray('0xff') + assert a * 8 == '0xffffffffffffffff' + assert 4 * a == '0xffffffff' + assert 1 * a == a * 1 == a + assert not a * 0 + a *= 3 + assert a == '0xffffff' + a *= 0 + assert not a + one = BitArray('0b1') + zero = BitArray('0b0') + assert one * 2 + 3 * zero + 2 * one * 2 == '0b110001111' + + a = BitArray(filename=os.path.join(THIS_DIR, 'test.m1v')) + length = a.len + a *= 3 + assert a.len == 3 * length + + with pytest.raises(ValueError): + _ = one * -1 + with pytest.raises(ValueError): + one *= -1 + with pytest.raises(ValueError): + _ = -1 * one + with pytest.raises(TypeError): + _ = one * 1.2 + with pytest.raises(TypeError): + _ = zero * one + + +@pytest.mark.parametrize( + ("left", "op", "right", "expected"), + [ + ('0b01101', '&', '0b00110', '0b00100'), + ('0b01101', '&', '0b11111', '0b01101'), + ('0b111001001', '|', '0b011100011', '0b111101011'), + ('0b111001001', '|', '0b000000000', '0b111001001'), + ('0b111001001', '^', '0b011100011', '0b100101010'), + ('0b111001001', '^', '0b111100000', '0b000101001'), + ('0xff00', '|', '0x00f0', '0xfff0'), + ('0o707', '^', '0o777', '0o070'), + ], +) +def test_bitwise_binary_operations(left, op, right, expected): + a = BitArray(left) + if op == '&': + result = a & right + elif op == '|': + result = a | right + else: + result = a ^ right + assert result == expected + + +def test_bitwise_errors_and_inplace(): + a = BitArray('0b01101') + with pytest.raises(ValueError): + _ = a & '0b1' + with pytest.raises(ValueError): + _ = a | '0b0000' + with pytest.raises(ValueError): + _ = a ^ '0b0000' + + a = BitArray.from_zeros(4) + with pytest.raises(ValueError): + a |= '0b111' + with pytest.raises(ValueError): + a &= '0b111' + with pytest.raises(ValueError): + a ^= '0b111' + + +def test_split_start_end_and_count_variants(): + a = BitArray('0b0 010100111 010100 0101 010') + subs = [i.bin for i in a.split('0b010')] + assert subs == ['0', '010100111', '010100', '0101', '010'] + + a = BitArray('0b000000') + bsl = a.split('0b1', bytealigned=False) + assert next(bsl) == a + with pytest.raises(StopIteration): + next(bsl) + b = BitArray() + bsl = b.split('0b001', bytealigned=False) + assert not next(bsl) + with pytest.raises(StopIteration): + _ = next(bsl) + + a = BitArray('0xaabbccbbccddbbccddee') + assert len(list(a.split('0xbb', bytealigned=True))) == 4 + bsl = list(a.split('0xbb', count=1, bytealigned=True)) + assert (len(bsl), bsl[0]) == (1, '0xaa') + bsl = list(a.split('0xbb', count=2, bytealigned=True)) + assert len(bsl) == 2 + assert bsl[0] == '0xaa' + assert bsl[1] == '0xbbcc' + + s = BitArray('0b1100011001110110') + for i in range(10): + assert list(s.split('0b11', bytealigned=False, count=i)) == list(s.split('0b11', bytealigned=False))[:i] + with pytest.raises(ValueError): + _ = next(s.split('0b11', count=-1)) + + +def test_split_start_end_boundaries(): + a = BitArray('0b0010101001000000001111') + bsl = a.split('0b001', bytealigned=False, start=1) + assert [x.bin for x in bsl] == ['010101', '001000000', '001111'] + with pytest.raises(ValueError): + _ = next(a.split('0b001', start=-100)) + with pytest.raises(ValueError): + _ = next(a.split('0b001', start=23)) + with pytest.raises(ValueError): + _ = next(a.split('0b1', start=10, end=9)) + + a = BitArray('0x00ffffee') + bsl = list(a.split('0b111', start=9, bytealigned=True)) + assert [x.bin for x in bsl] == ['1111111', '11111111', '11101110'] + + a = BitArray('0b000010001001011') + bsl = list(a.split('0b1', bytealigned=False, end=14)) + assert [x.bin for x in bsl] == ['0000', '1000', '100', '10', '1'] + assert list(a[4:12].split('0b0', bytealigned=False)) == list(a.split('0b0', start=4, end=12)) + + +def test_find_start_end_boundaries(): + a = Reader(BitArray('0b0010000100')) + found = a.find('0b1', start=4) + assert (found, a.bitpos) == (7, 7) + found = a.find('0b1', start=2) + assert (found, a.bitpos) == (2, 2) + found = a.find('0b1', bytealigned=False, start=8) + assert (found, a.bitpos) == (None, 2) + + a = Reader(BitArray('0b0010010000')) + found = a.find('0b1', bytealigned=False, end=2) + assert (found, a.bitpos) == (None, 0) + found = a.find('0b1', end=3) + assert (found, a.bitpos) == (2, 2) + found = a.find('0b1', bytealigned=False, start=3, end=5) + assert (found, a.bitpos) == (None, 2) + found = a.find('0b1', start=3, end=6) + assert (found, a.bitpos) == (5, 5) + + b = Reader(BitArray('0x0011223344')) + with pytest.raises(ValueError): + _ = b.find('0x22', bytealigned=True, start=-100) + with pytest.raises(ValueError): + _ = b.find('0x22', end=41, bytealigned=True) + + +def test_findall_generator_and_count(): + a = BitArray('0xff1234512345ff1234ff12ff') + p = a.findall('0xff', bytealigned=True) + assert next(p) == 0 + assert next(p) == 6 * 8 + assert next(p) == 9 * 8 + assert next(p) == 11 * 8 + with pytest.raises(StopIteration): + _ = next(p) + + s = BitArray('0b1') * 100 + for i in [0, 1, 23]: + assert len(list(s.findall('0b1', count=i))) == i + with pytest.raises(ValueError): + _ = s.findall('0b1', bytealigned=True, count=-1) + + +def test_set_reset_properties(): + s = BitArray() + s.hex = '0' + assert s.hex == '0' + s.hex = '0x010203045' + assert s.hex == '010203045' + with pytest.raises(bitstring.CreationError): + s.hex = '0x002g' + + s = BitArray(bin="000101101") + assert s.bin == '000101101' + assert s.len == 9 + s.bin = '0' + assert s.bin == '0' + assert s.len == 1 + + s = BitArray(hex='0x000001b3') + s.bin = '' + assert s.len == 0 + assert s.bin == '' + + +def test_overwrite_more_cases(): + s = BitArray(hex='342563fedec') + s2 = BitArray(s) + s.overwrite(BitArray(bin=''), 23) + assert s.bin == s2.bin + + s = BitArray('0x123') + s.overwrite(s, 0) + assert s == '0x123' + + s = BitArray(bin='11111') + with pytest.raises(ValueError): + s.overwrite(BitArray(bin='1'), -10) + with pytest.raises(ValueError): + s.overwrite(BitArray(bin='1'), 6) + s.overwrite('bin=0', 5) + assert s.bin == '111110' + s.overwrite(BitArray(hex='0x00'), 1) + assert s.bin == '100000000' + + +def test_join_more_cases(): + s1 = BitArray(hex='00112233445566778899aabbccddeeff') + s2 = BitArray(bin='0b000011') + bsl = [s1[0:32], s1[4:12], s2, s2, s2, s2] + assert BitArray().join(bsl).hex == '00112233010c30c3' + + bsl = [BitArray(uint=j, length=12) for j in range(10) for _ in range(10)] + assert BitArray().join(bsl).length == 1200 + + with pytest.raises(TypeError): + _ = BitArray().join([1, 2]) + + a = BitArray().join(['0xa', '0xb', '0b1111']) + assert a == '0xabf' + a = BitArray('0b1').join(['0b0' for _ in range(10)]) + assert a == '0b0101010101010101010' + assert not BitArray('0xff').join([]) + a = BitArray('0xff').join([Bits.from_zeros(5), '0xab', '0xabc']) + assert a == '0b00000, 0xffabffabc' + + +def test_file_object_creation_and_copy(): + filename = os.path.join(THIS_DIR, 'test.m1v') + with open(filename, 'rb') as f: + s = Bits.from_file(f, offset=32, length=12) + assert s.uint == 352 + t = Bits('0xf') + f + assert t.startswith('0xf000001b3160') + s2 = Bits.from_file(f) + t2 = BitArray('0xc') + t2.prepend(s2) + assert t2.startswith('0x000001b3') + assert t2.endswith('0xc') + with open(filename, 'rb') as b: + u = BitArray(bytes=b.read()) + assert u == s2 + + with open(os.path.join(THIS_DIR, 'smalltestfile'), 'rb') as f: + s = BitArray.from_file(f) + t = BitArray(s) + s.prepend('0b1') + assert s[1:] == t + s = BitArray.from_file(f) + t = copy.copy(s) + t.append('0b1') + assert s == t[:-1] + + +def test_big_little_endian_error_cases(): + with pytest.raises(bitstring.CreationError): + _ = BitArray(ube=100, length=15) + with pytest.raises(bitstring.CreationError): + _ = BitArray(ibe=100, length=15) + with pytest.raises(bitstring.CreationError): + _ = BitArray('ube:17=100') + with pytest.raises(bitstring.CreationError): + _ = BitArray('ibe:7=2') + + s = Reader(BitArray('0b1')) + with pytest.raises(bitstring.InterpretError): + _ = s.bits.ibe + with pytest.raises(bitstring.InterpretError): + _ = s.bits.ube + with pytest.raises(ValueError): + _ = s.read('ube') + with pytest.raises(ValueError): + _ = s.read('ibe') + + with pytest.raises(bitstring.CreationError): + _ = BitArray('ule:15=10') + with pytest.raises(bitstring.CreationError): + _ = BitArray('ile:31=-999') + with pytest.raises(bitstring.CreationError): + _ = BitArray(ule=100, length=15) + with pytest.raises(bitstring.CreationError): + _ = BitArray(ile=100, length=15) + + +def test_little_endian_int_cases(): + s = BitArray(int=100, length=16) + assert s.ile == 25600 + s = BitArray(ile=100, length=16) + assert s.int == 25600 + assert s.ile == 100 + s.ile = 105 + assert s.ile == 105 + s = BitArray('ile:32=999') + assert s.ile == 999 + s.byteswap() + assert s.int == 999 + s = pack('ile:24', 1001) + assert s.ile == 1001 + assert Reader(s).read('ile24') == 1001 + + +@pytest.mark.parametrize( + ("fmt", "value", "expected"), + [ + ('b', 23, 'ibe:8=23'), + ('>B', 23, 'ube:8=23'), + ('>h', 23, 'ibe:16=23'), + ('>H', 23, 'ube:16=23'), + ('>l', 23, 'ibe:32=23'), + ('>L', 23, 'ube:32=23'), + ('>i', 23, 'ibe:32=23'), + ('>I', 23, 'ube:32=23'), + ('>q', 23, 'ibe:64=23'), + ('>Q', 23, 'ube:64=23'), + ], +) +def test_struct_token_matrix(fmt, value, expected): + assert pack(fmt, value) == BitArray(expected) + + +def test_struct_token_multiplicative_factors_and_errors(): + s = pack('<2h', 1, 2) + assert s.unpack('<2h') == [1, 2] + s = pack('<100q', *range(100)) + assert s.len == 100 * 64 + assert s[44 * 64:45 * 64].ule == 44 + s = pack('@L0B2h', 5, 5, 5) + assert s.unpack('@Lhh') == [5, 5, 5] + + for f in ['>>q', '<>q', 'q>', '2q', 'q', '>-2q', '@a', '>int:8', '>q2']: + with pytest.raises(bitstring.CreationError): + _ = pack(f, 100) + + +def test_byteswap_int_pack_code_iterable_and_errors(): + s = BitArray(pack('5*ule:16', *range(10, 15))) + assert list(range(10, 15)) == s.unpack('5*ule:16') + swaps = s.byteswap(2) + assert list(range(10, 15)) == s.unpack('5*ube:16') + assert swaps == 5 + + s = BitArray('0xf234567f') + swaps = s.byteswap(1, start=4) + assert swaps == 3 + assert s == '0xf234567f' + s.byteswap(2, start=4) + assert s == '0xf452367f' + s.byteswap(2, start=4, end=-4) + assert s == '0xf234567f' + s.byteswap(3) + assert s == '0x5634f27f' + s.byteswap(2, repeat=False) + assert s == '0x3456f27f' + + s = BitArray('0x0011223344556677') + assert s.byteswap('b') == 8 + assert s == '0x0011223344556677' + assert s.byteswap('>3h', repeat=False) == 1 + assert s == '0x1100332255446677' + + s = BitArray('0x0011223344556677') + assert s.byteswap(range(1, 4), repeat=False) == 1 + assert s == '0x0022115544336677' + assert s.byteswap([2], start=8) == 3 + assert s == '0x0011224455663377' + + with pytest.raises(ValueError): + s.byteswap('z') + with pytest.raises(ValueError): + s.byteswap(-1) + with pytest.raises(ValueError): + s.byteswap([-1]) + with pytest.raises(ValueError): + s.byteswap([1, 'e']) + with pytest.raises(TypeError): + s.byteswap(5.4) + + +@pytest.mark.parametrize( + ("value", "prefix", "args", "expected"), + [ + ('0x12ff', '0x1', (), True), + ('0x12ff', '0b0001001', (), True), + ('0x12ff', '0x12ff', (), True), + ('0x12ff', '0x12ff, 0b1', (), False), + ('0x123456', '0x234', (4,), True), + ('0x123456', '0x123', (None, 11), False), + ('0x123456', '0x123', (None, 12), True), + ('0x123456', '0x34', (8, 16), True), + ], +) +def test_startswith_cases(value, prefix, args, expected): + a = BitArray(value) + if not args: + assert a.startswith(prefix) is expected + elif len(args) == 1: + assert a.startswith(prefix, start=args[0]) is expected + else: + assert a.startswith(prefix, start=args[0], end=args[1]) is expected + + +@pytest.mark.parametrize( + ("value", "suffix", "args", "expected"), + [ + ('0xf2341', '0x41', (), True), + ('0xf2341', '0b001', (), True), + ('0xf2341', '0xf2341', (), True), + ('0xf2341', '0x1f2341', (), False), + ('0x123456', '0x234', (None, 16), True), + ('0x123456', '0x456', (13,), False), + ('0x123456', '0x456', (12,), True), + ('0x123456', '0x34', (8, 16), True), + ], +) +def test_endswith_cases(value, suffix, args, expected): + a = BitArray(value) + if not args: + assert a.endswith(suffix) is expected + elif len(args) == 1: + assert a.endswith(suffix, start=args[0]) is expected + else: + assert a.endswith(suffix, start=args[0], end=args[1]) is expected + + +def test_read_unpack_peek_with_keyword_lengths(): + a = Bits('0xff, 0b000, 0xf') + x, y, z = a.unpack('hex:a, bin, hex:b', a=8, b=4) + assert x == 'ff' + assert y == '000' + assert z == 'f' + + a = Bits('0b110') + x, = a.unpack('bin:3', notused=33) + assert x == '110' + + a = pack('uint:p=33', p=12) + with pytest.raises(ValueError): + a.unpack('uint:p') + with pytest.raises(ValueError): + a.unpack('uint:p', p='a_string') + + s = Reader(BitArray('0x0102')) + x, y = s.peeklist('8, hex:b', b=4) + assert (x, y) == ('0x01', '0') + assert s.pos == 0 + + +def test_bytes_keyword_problem(): + s = Bits('0x01') + x, = s.unpack('bytes:a', a=1) + assert x == b'\x01' + + s = Bits('0x000ff00a') + x, y, z = s.unpack('12, bytes:x, bits', x=2) + assert (x.int, y, z) == (0, b'\xff\x00', '0xa') + + +def test_bool_assignment_and_errors(): + a = BitArray() + a.bool = True + assert a.bool is True + a.hex = 'ee' + a.bool = False + assert a.bool is False + a.bool = 'False' + assert a.bool is False + a.bool = 'True' + assert a.bool is True + a.bool = 0 + assert a.bool is False + a.bool = 1 + assert a.bool is True + + for args in [('bool=true',), ('True',), ('bool', 2)]: + with pytest.raises(bitstring.CreationError): + _ = pack(*args) + with pytest.raises(bitstring.InterpretError): + _ = BitArray('0b11').bool + b = BitArray() + with pytest.raises(bitstring.InterpretError): + _ = b.bool + with pytest.raises(bitstring.CreationError): + b.bool = 'false' + + a = Reader(Bits('0xf')) + with pytest.raises(ValueError): + _ = a.read('bool:0') + with pytest.raises(ValueError): + _ = a.read('bool:2') + + +def test_zero_bit_reads_and_read_int_list(): + a = Reader(Bits('0x123456')) + with pytest.raises(bitstring.InterpretError): + _ = a.read('uint:0') + with pytest.raises(bitstring.InterpretError): + _ = a.read('float:0') + + a = Reader(Bits('0xab, 0b110')) + b, c = a.readlist([8, 3]) + assert b.hex == 'ab' + assert c.bin == '110' + + +def test_count_with_offset_and_bytes_problem_cases(): + a = Bits('0xff0120ff') + b = a[1:-1] + assert b.count(1) == 16 + assert b.count(0) == 14 + + b = BitArray(bytes=b'\x00\xaa', offset=8) + assert b.hex == 'aa' + b = BitArray(bytes=b'\x00\xaa', offset=4) + assert b.hex == '0aa' + b = BitArray(bytes=b'\x00\xaa', offset=8, length=8) + b.invert() + assert b.hex == '55' + b = BitArray(bytes=b'\xaa\xbb', offset=8, length=4) + b.prepend('0xe') + assert b.hex == 'eb' + b = BitArray(bytes=b'\x01\x02\x03\x04', offset=8) + b.byteswap() + assert b == '0x040302' + + +def test_format_and_cacheing_cases(): + a = Bits('0xabc') + assert f'{a}' == '0xabc' + a += '0b0' + assert f'{a}' == '0b1010101111000' + b = BitArray.from_zeros(10) + assert f'{b}' == '0b0000000000' + c = BitArray(filename=os.path.join(THIS_DIR, 'test.m1v')) + assert f'{c}'[0:10] == '0x000001b3' + assert f'{Bits("0xf").bin}' == '1111' + + _ = BitArray('0xdeadbeef1000') + with pytest.raises(bitstring.CreationError): + _ = BitArray('0xdeadbeef1000', offset=8) + _ = BitArray('0xdeadbeef002') + with pytest.raises(bitstring.CreationError): + _ = BitArray('0xdeadbeef002', length=16) + + +def test_operator_identity_semantics_for_bits_and_bitarray(): + a1 = Bits('0xabc') + b1 = a1 + a1 += '0xdef' + assert a1 == '0xabcdef' + assert b1 == '0xabc' + + a2 = BitArray('0xabc') + b2 = a2 + c2 = a2 + '0x0' + a2 += '0xdef' + assert a2 == '0xabcdef' + assert b2 == '0xabcdef' + assert c2 == '0xabc0' + + a1 = Bits('0xabc') + b1 = a1 + a1 &= '0xf0f' + assert a1 == '0xa0c' + assert b1 == '0xabc' + + a2 = BitArray('0xabc') + b2 = a2 + c2 = a2 & '0x00f' + a2 &= '0xf0f' + assert a2 == '0xa0c' + assert b2 == '0xa0c' + assert c2 == '0x00c' + + +def test_rotation_file_and_errors(): + a = BitArray() + with pytest.raises(bitstring.Error): + a.ror(0) + a += '0b001' + with pytest.raises(ValueError): + a.ror(-1) + + a = BitArray() + with pytest.raises(bitstring.Error): + a.rol(0) + a += '0b001' + with pytest.raises(ValueError): + a.rol(-1) + + a = BitArray(filename=os.path.join(THIS_DIR, 'test.m1v')) + m = a.len + a.rol(1) + assert a.startswith('0x000003') + assert a.len == m + assert a.endswith('0x0036e') + + a = BitArray(filename=os.path.join(THIS_DIR, 'test.m1v')) + m = a.len + a.ror(1) + assert a.startswith('0x800000') + assert a.len == m + assert a.endswith('0x000db') + + +def test_efficient_overwrite_and_large_counts(): + a = BitArray.from_zeros(1000000) + a.overwrite([1], 123456) + assert a[123456] is True + a.overwrite('0xff', 1) + assert a[0:32:1] == '0x7f800000' + + c = BitArray(length=1000) + c.overwrite('0xaaaaaaaaaaaa', 81) + assert c[81:81 + 6 * 8] == '0xaaaaaaaaaaaa' + assert len(list(c.findall('0b1'))) == 24 + s = BitArray(length=1000)[5:] + s.overwrite('0xffffff', 500) + r = Reader(s, pos=500) + assert r.read(4 * 8) == '0xffffff00' + s.overwrite('0xff', 502) + assert s[502:518] == '0xffff' + + +def test_find_remaining_corner_cases(): + s = Reader(BitArray('0xff')) + assert s.find(s.bits) == 0 + assert s.find(BitArray(hex='0x12')) is None + assert s.find(BitArray(hex='0xffff')) is None + + s = Reader(BitArray(hex='0x1122334455'), pos=2) + s.find('0x66', bytealigned=True) + assert s.pos == 2 + s.pos = 38 + s.find('0x66', bytealigned=True) + assert s.pos == 38 + + s = Reader(BitArray('0x1234')) + assert s.find('0x1234') == 0 + assert s.find('0x1234') == 0 + s.bits.append('0b111') + s.pos = 3 + assert s.find('0b1', start=17, bytealigned=True) is None + assert s.pos == 3 + + +def test_rfind_endbit_cases(): + a = Reader(BitArray('0x000fff')) + b = a.rfind('0b011', start=0, end=14, bytealigned=False) + assert b is not None + assert a.rfind('0b011', start=0, end=13, bytealigned=False) is None + + +def test_remaining_shift_in_place_errors_and_whole_length(): + s = BitArray('0xabcd') + s >>= len(s) + assert s == '0x0000' + + s = BitArray() + with pytest.raises(ValueError): + s >>= 1 + s += '0b11' + with pytest.raises(ValueError): + s >>= -1 + + s = BitArray() + with pytest.raises(ValueError): + s <<= 1 + s += '0b11' + with pytest.raises(ValueError): + s <<= -1 + + +def test_replace_range_cases(): + a = BitArray('0x00114723ef4732344700') + n = a.replace('0x47', '0x00', bytealigned=True) + assert n == 3 + assert a.hex == '00110023ef0032340000' + a.replace('0x00', '', bytealigned=True) + assert a.hex == '1123ef3234' + a.replace('0x11', '', start=1, bytealigned=True) + assert a.hex == '1123ef3234' + a.replace('0x11', '0xfff', end=7, bytealigned=True) + assert a.hex == '1123ef3234' + a.replace('0x11', '0xfff', end=8, bytealigned=True) + assert a.hex == 'fff23ef3234' + + a = BitArray.from_string('0xab') + b = BitArray.from_string('0xcd') + c = BitArray.from_string('0xabef') + c.replace(a, b) + assert c == '0xcdef' + assert a == '0xab' + assert b == '0xcd' + + +def test_pack_uint_and_default_uint_errors(): + s = pack('uint:10, uint:5', 1, 2) + a, b = s.unpack('10, 5') + assert (a.uint, b.uint) == (1, 2) + s = pack('uint:10=150, uint:12=qee', qee=3) + assert s == 'uint:10=150, uint:12=3' + assert BitArray('uint:100=5') == 'uint:100=5' + with pytest.raises(bitstring.CreationError): + _ = BitArray('5=-1') + + +def test_packing_long_keyword_and_variable_lengths(): + s = pack('bits=b', b=BitArray.from_zeros(128000)) + assert s == BitArray.from_zeros(128000) + with pytest.raises(bitstring.CreationError): + _ = pack('bin:1') + with pytest.raises(bitstring.CreationError): + _ = pack('', 100) + assert pack('uint10', uint10='0b1') == '0b1' + assert pack('0b110', **{'0b110': '0xfff'}) == '0xfff' + for i in range(1, 11): + assert pack('uint:n', 0, n=i).bin == '0' * i + + +def test_pack_capital_keywords_and_other_capitals(): + assert pack('A', A='0b1') == '0b1' + format_ = 'bits:4=BL_OFFT, uint:12=width, uint:12=height' + d = {'BL_OFFT': '0b1011', 'width': 352, 'height': 288} + assert bitstring.pack(format_, **d) == '0b1011, uint:12=352, uint:12=288' + assert pack('0X0, uint:8, hex', 45, '0XABcD') == '0x0, uint:8=45, 0xabCD' + assert Bits('0XABC, 0O0, 0B11') == 'hex=0Xabc, oct=0, bin=0B11' + + +def test_file_slices_errors_and_hex_reset(): + filename = os.path.join(THIS_DIR, 'smalltestfile') + s = BitArray(filename=filename) + assert s[-16:].hex == 'cdef' + with pytest.raises(IOError): + _ = BitArray(filename='Idonotexist') + + s = BitArray(filename=os.path.join(THIS_DIR, 'test.m1v')) + assert s[0:32].hex == '000001b3' + assert s[-32:].hex == '000001b7' + s.hex = '0x11' + assert s.hex == '11' + + +def test_file_reader_independent_positions(): + filename = os.path.join(THIS_DIR, 'test.m1v') + s1 = Reader(BitArray(filename=filename)) + s2 = Reader(BitArray(filename=filename)) + assert s1.read(32).hex == '000001b3' + assert s2.read(32).hex == '000001b3' + s1.bytepos += 4 + assert s1.read(8).hex == '02' + assert s2.read(5 * 8).hex == '1601208302' + s1.pos = len(s1) + with pytest.raises(ValueError): + s1.pos += 1 + s1.read(0) + + +def test_insert_null_bits_and_self(): + s = BitArray(hex='0x123') + s.insert(BitArray(), 3) + assert s.hex == '123' + + one = BitArray(bin='1') + zero = BitArray(bin='0') + s = BitArray(bin='00') + s.insert(one, 0) + assert s.bin == '100' + s.insert(zero, 0) + assert s.bin == '0100' + s.insert(one, s.len) + assert s.bin == '01001' + s.insert(s, 2) + assert s.bin == '0101001001' + + +def test_more_adding_radd_and_self_append(): + s = BitArray(bin='00') + BitArray(bin='') + BitArray(bin='11') + assert s.bin == '0011' + s = '0b01' + s += BitArray('0b11') + assert s.bin == '0111' + s = BitArray('0x00') + t = BitArray('0x11') + s += t + assert s.hex == '0011' + assert t.hex == '11' + s += s + assert s.hex == '00110011' + assert ('0xff' + BitArray('0xee')).hex == 'ffee' + + +def test_delete_bits_bytes_and_getitems(): + s = BitArray(bin='000111100000') + del s[4:8] + assert s.bin == '00010000' + del s[4:1004] + assert s.bin == '0001' + + s = BitArray('0x00112233') + del s[8:8] + assert s.hex == '00112233' + del s[8:16] + assert s.hex == '002233' + del s[:24] + assert not s + + s = BitArray(bin='0b1011') + assert [s[i] for i in range(4)] == [True, False, True, True] + with pytest.raises(IndexError): + _ = s[4] + assert [s[i] for i in range(-1, -5, -1)] == [True, True, False, True] + with pytest.raises(IndexError): + _ = s[-5] + + +def test_slicing_and_negative_slicing_more(): + s = Bits(hex='0123456789') + assert s[0:8].hex == '01' + assert not s[0:0] + assert not s[23:20] + assert s[8:12].bin == '0010' + assert s[32:80] == '0x89' + + s = Bits(hex='012345678') + assert s[:-8].hex == '0123456' + assert s[-16:-8].hex == '56' + assert s[-24:].hex == '345678' + assert s[-1000:-24] == '0x012' + + +def test_writing_data_and_offsets(): + strings = [BitArray(bin=x) for x in ['0', '001', '0011010010', '010010', '1011']] + s = BitArray().join(strings) + s2 = BitArray(bytes=s.bytes) + assert s2.bin == '000100110100100100101011' + s2.append(BitArray(bin='1')) + s3 = BitArray(bytes=s2.tobytes()) + assert s3.bin == '00010011010010010010101110000000' + + s1 = BitArray(bytes=b'\x10') + s2 = BitArray(bytes=b'\x08\x00', length=8, offset=1) + s3 = BitArray(bytes=b'\x04\x00', length=8, offset=2) + assert s1 == s2 == s3 + assert s1.bytes == s2.bytes == s3.bytes + + +def test_various_compositions_with_reader(): + hexes = ['12345678', '87654321', 'ffffffffff', 'ed', '12ec'] + bins = ['001010', '1101011', '0010000100101110110110', '11', '011'] + bsl = [] + for (hex_, bin_) in list(zip(hexes, bins)) * 5: + bsl.append(BitArray(hex=hex_)) + bsl.append(BitArray(bin=bin_)) + r = Reader(BitArray().join(bsl)) + for (hex_, bin_) in list(zip(hexes, bins)) * 5: + h = r.read(4 * len(hex_)) + b = r.read(len(bin_)) + assert h.hex == hex_ + assert b.bin == bin_ + + s1 = BitArray(hex="0x1f08")[:13] + assert s1.bin == '0001111100001' + s2 = BitArray(bin='0101') + s1.append(s2) + assert s1.bin == '00011111000010101' + assert s1[3:8].bin == '11111' + + +def test_reader_position_arithmetic(): + s = Reader(BitArray(hex='0xff'), pos=6) + s.pos += 1 + assert s.bitpos == 7 + s.bitpos += 1 + assert s.pos == 8 + s.pos += 1 + with pytest.raises(ValueError): + s.read(0) + + s = Reader(BitArray(hex='0x010203')) + s.bytepos += 1 + assert s.bytepos == 1 + s.bytepos += 1 + assert s.bytepos == 2 + s.bytepos += 1 + assert s.bytepos == 3 + s.bytepos += 1 + with pytest.raises(ValueError): + s.read(0) + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ('bin=001', '0b001'), + ('oct=0o007', '0o007'), + ('hex=123abc', '0x123abc'), + ('bin2=01', '0b01'), + ('0xff 0Xee 0xd 0xcc', '0xffeedcc'), + ('0b0 0B111 0b001', '0b0111001'), + (' 0o123 0O 7 0 o1', '0o12371'), + ], +) +def test_auto_creation_more(source, expected): + assert BitArray(source) == expected + + +@pytest.mark.parametrize("source", ['bin:1=01', 'bits:4=0b1', 'oct3=000', 'hex4=0x1234']) +def test_auto_creation_error_more(source): + with pytest.raises(bitstring.CreationError): + _ = BitArray(source) + + +def test_more_auto_methods_and_split_with_self(): + s = BitArray('0b000000010100011000') + assert Reader(s).find('0b101') == 7 + s = BitArray('0x00004700') + r = Reader(s) + assert r.find('0b01000111', bytealigned=True) == 16 + assert r.bytepos == 2 + + s = BitArray('0x000143563200015533000123') + sections = s.split('0x0001') + assert next(sections).hex == '' + assert next(sections).hex == '0001435632' + assert next(sections).hex == '00015533' + assert next(sections).hex == '000123' + with pytest.raises(StopIteration): + next(sections) + + s = BitArray('0x1234') + sections = s.split(s) + assert next(sections).hex == '' + assert next(sections).hex == '1234' + with pytest.raises(StopIteration): + next(sections) + + +def test_multiple_auto_mutations_and_reverse(): + s = BitArray('0xa') + s.prepend('0xf') + s.append('0xb') + assert s == '0xfab' + s.prepend(s) + s.append('0x100') + s.overwrite('0x5', 4) + assert s == '0xf5bfab100' + + s = BitArray('0b0011') + s.reverse() + assert s.bin == '1100' + s = BitArray('0b10') + s.reverse() + assert s.bin == '01' + s = BitArray() + s.reverse() + assert s.bin == '' + + +def test_equality_not_equals_and_auto_copy(): + s1 = BitArray('0b01010101') + s2 = BitArray('0b01010101') + assert s1 == s2 + assert BitArray() == BitArray() + s5 = BitArray(bytes=b'\xff', offset=2, length=3) + assert s5 == BitArray('0b111') + assert s5 != object() + + assert BitArray('0b0') != BitArray('0b1') + assert not BitArray('0b0') != BitArray('0b0') + + a = BitArray('0b00110111') + assert a == '0b00110111' + assert a == '0x37' + assert '0b0011 0111' == a + assert '0x3 0x7' == a + assert a != '0b11001000' + + s = BitArray('0xabcdef') + t = BitArray(s) + assert t.hex == 'abcdef' + del s[-8:] + assert t.hex == 'abcdef' + + +def test_invert_special_method_cases(): + s = BitArray('0b00011001') + t = ~s + assert t.bin == '11100110' + assert (~BitArray('0b0')).bin == '1' + assert (~BitArray('0b1')).bin == '0' + assert ~t == s + with pytest.raises(bitstring.Error): + _ = ~BitArray() + + +def test_large_equals_and_mutation(): + s1 = BitArray.from_zeros(1000000) + s2 = BitArray.from_zeros(1000000) + s1.set(True, [-1, 55, 53214, 534211, 999999]) + s2.set(True, [-1, 55, 53214, 534211, 999999]) + assert s1 == s2 + s1.set(True, 800000) + assert s1 != s2 + + +def test_cut_problem_and_more_cut_errors(): + s = BitArray('0x1234') + for n in list(s.cut(4)): + s.prepend(n) + assert s == '0x43211234' + + a = BitArray('0b1') + with pytest.raises(ValueError): + _ = next(a.cut(1, start=1, end=2)) + with pytest.raises(ValueError): + _ = next(a.cut(1, start=-2, end=1)) + with pytest.raises(ValueError): + _ = next(a.cut(0)) + with pytest.raises(ValueError): + _ = next(a.cut(1, count=-1)) + + +def test_token_parser_struct_codes(): + tp = bitstring.utils.tokenparser + assert tp('>H') == (False, [('ube', 16, None)]) + assert tp('b') == (False, [('i', 8, None)]) + assert tp('>= 4 + assert a1 == '0x0ab' + assert b1 == '0xabc' + + +def test_function_negative_indices_more_cases(): + s = BitArray('0x1234151f') + assert list(s.findall('0x1', bytealigned=True, start=-15)) == [24] + assert list(s.findall('0x1', bytealigned=True, start=-16)) == [16, 24] + assert list(s.findall('0x1', bytealigned=True, end=-5)) == [0, 16] + assert list(s.findall('0x1', bytealigned=True, end=-4)) == [0, 16, 24] + + r = Reader(s) + assert r.rfind('0x1f', end=-1) is None + assert r.rfind('0x12', start=-31) is None + + s = BitArray('0x12345') + assert list(s.cut(4, start=-12, end=-4)) == ['0x3', '0x4'] + + s = BitArray('0xfe0012fe1200fe') + assert list(s.split('0xfe', bytealigned=True, end=-1)) == ['', '0xfe0012', '0xfe1200f, 0b111'] + assert list(s.split('0xfe', bytealigned=True, start=-8)) == ['', '0xfe'] + assert s.startswith('0x00f', start=-16) + assert s.startswith('0xfe00', end=-40) + assert not s.startswith('0xfe00', end=-41) + assert s.endswith('0x00fe', start=-16) + assert not s.endswith('0x00fe', start=-15) + assert not s.endswith('0x00fe', end=-1) + assert s.endswith('0x00f', end=-4) + + s.replace('0xfe', '', end=-1) + assert s == '0x00121200fe' + s.replace('0x00', '', start=-24) + assert s == '0x001212fe' + + +def test_rotate_start_and_end_more_cases(): + a = BitArray('0b110100001') + a.rol(1, start=3, end=6) + assert a == '0b110001001' + a.ror(1, start=-4) + assert a == '0b110001100' + a.rol(202, end=-5) + assert a == '0b001101100' + a.ror(3, end=4) + assert a == '0b011001100' + with pytest.raises(ValueError): + a.rol(5, start=-4, end=-6) + + +def test_byte_swap_from_file_and_remaining_iterable_case(): + s = BitArray(filename=os.path.join(THIS_DIR, 'smalltestfile')) + swaps = s.byteswap('2bh') + assert s == '0x0123674589abefcd' + assert swaps == 2 + + s = BitArray('0x0011223344556677') + s.byteswap(range(1, 4), repeat=False) + s.byteswap([2], start=8) + assert s.byteswap([2, 3], start=4) == 1 + assert s == '0x0120156452463377' + + +def test_bracket_tokens_and_packing_default_int_keyword(): + s = BitArray('3*(0x0, 0b1)') + assert s == '0x0, 0b1, 0x0, 0b1, 0x0, 0b1' + s = pack('2*(uint:12, 3*(uint:7, uint:6))', *range(3, 17)) + a = s.unpack('12, 7, 6, 7, 6, 7, 6, 12, 7, 6, 7, 6, 7, 6') + assert [x.uint for x in a] == list(range(3, 17)) + assert a == s.unpack('2*(12,3*(7,6))') + + s = pack('uint:12', 100) + assert s.unpack('12')[0].uint == 100 + s = pack('int:oh_no_not_the_eyes=33', oh_no_not_the_eyes=17) + assert s.int == 33 + assert s.len == 17 + + +def test_hash_edge_cases_for_bits(): + a = Bits('0xabcd') + b = Bits('0xabcd') + c = b[1:] + assert hash(a) == hash(b) + assert hash(a) != hash(c) + s = {Bits(uint=1 << 300, length=10000), Bits(uint=2 << 300, length=10000), Bits(uint=3 << 300, length=10000)} + assert len(s) == 3 + + +def test_bits_function_return_types_and_copy(): + s = Bits('0xf, 0b1') + t = copy.copy(s) + assert type(t) is Bits + a = s + '0o3' + assert type(a) is Bits + assert type(a[0:4]) is Bits + assert type(a[4:3]) is Bits + assert type(a[5:2:-1]) is Bits + assert type(~a) is Bits + assert type(a << 2) is Bits + assert type(a >> 2) is Bits + assert type(a * 2) is Bits + assert type(a & ~a) is Bits + assert type(Reader(a).read(4)) is Bits + + +def test_bits_property_immutability_more(): + a = Bits('0x123123') + with pytest.raises(AttributeError): + a.hex = '0x234' + with pytest.raises(AttributeError): + a.oct = '0o234' + with pytest.raises(AttributeError): + a.bin = '0b101' + with pytest.raises(AttributeError): + a.ue = 3453 + with pytest.raises(AttributeError): + a.se = -123 + with pytest.raises(AttributeError): + a.int = 432 + with pytest.raises(AttributeError): + a.bytes = b'hello' + + +def test_bytes_problem_more_cases(): + b = BitArray(bytes=b'\x00\xaa', offset=8, length=4) + assert b.bin == '1010' + b = BitArray(bytes=b'\x00\xaa', offset=8, length=8) + b.prepend('0xee') + assert b.hex == 'eeaa' + + +def test_add_empty_bits_issue_more(): + x = BitArray() + assert x + Bits('0xff') == '0xff' + assert x + bitstring.BitArray('0xff') == '0xff' + xx = Bits() + assert xx + Bits('0xff') == '0xff' + assert xx + bitstring.BitArray('0xff') == '0xff' + + +@pytest.mark.parametrize( + ("fmt", "values", "expected_fmt"), + [ + ('=B', (23,), 'une:8=23'), + ('=h', (23,), 'ine:16=23'), + ('=H', (23,), 'une:16=23'), + ('@l', (23,), 'ine:32=23'), + ('@L', (23,), 'une:32=23'), + ('@i', (23,), 'ine:32=23'), + ('@I', (23,), 'une:32=23'), + ('@q', (23,), 'ine:64=23'), + ('@Q', (23,), 'une:64=23'), + ], +) +def test_native_struct_pack_codes_more(fmt, values, expected_fmt): + assert pack(fmt, *values) == BitArray(expected_fmt) + + +@pytest.mark.parametrize("source", [bytearray(b'uint:5=2'), memoryview(b'uint:5=2')]) +def test_buffer_creation_more(source): + bits = Bits(source) + assert len(bits) >= 0 + + +@pytest.mark.parametrize("source", [range(12), (0, 1, '0', '1'), [True, False, [], [0], 'hello']]) +def test_bool_iterable_factory_more(source): + bits = Bits.from_bools(source) + assert len(bits) >= 0 + + +@pytest.mark.parametrize("source", ['5', '+0.0001', '-1e101', '4.', '.2', '-.65', '43.21E+32']) +def test_float64_init_strings_more(source): + a = BitArray('float:64=%s' % source) + assert a.float == float(source) + + +@pytest.mark.parametrize("source", ['5', '+0.5', '-1e2', '4.', '.25', '-.75']) +def test_float16_init_strings_more(source): + a = BitArray('float:16=%s' % source) + assert a.f == float(source) + + +@pytest.mark.parametrize("token", ['hex:1', 'oct:1', '-5', 'fred', 'bin:-2', 'uint:p', + 'uint:-2', 'int:u', 'int:-3', 'ses', 'uee', '-14']) +def test_invalid_reader_tokens_more(token): + with pytest.raises(ValueError): + _ = Reader(BitArray('0x123456')).read(token) + + +@pytest.mark.parametrize( + "kwargs", + [ + {'ube': 100, 'length': 15}, + {'ibe': 100, 'length': 15}, + {'ule': 100, 'length': 15}, + {'ile': 100, 'length': 15}, + {'fle': 0.3, 'length': 0}, + {'fle': 0.3, 'length': 1}, + {'float': 2}, + ], +) +def test_creation_error_kwargs_more(kwargs): + with pytest.raises(bitstring.CreationError): + _ = BitArray(**kwargs) + + +@pytest.mark.parametrize( + "source", + [ + 'ube:17=100', + 'ibe:7=2', + 'ule:15=10', + 'ile:31=-999', + 'bool=true', + 'True', + 'hello', + ], +) +def test_creation_error_strings_more(source): + with pytest.raises(bitstring.CreationError): + _ = BitArray(source) + + +@pytest.mark.parametrize("fmt", ['>>q', '<>q', 'q>', '2q', 'q', '>-2q', '@a', '>int:8', '>q2']) +def test_struct_token_error_cases_more(fmt): + with pytest.raises(bitstring.CreationError): + _ = pack(fmt, 100) + + +@pytest.mark.parametrize("fmt", [' str: - colour_escape = re.compile(r'(?:\x1B[@-_])[0-?]*[ -/]*[@-~]') - return colour_escape.sub('', s) - - -@settings(max_examples=500) -@given(length=st.integers(0, 9), - start=st.integers(-20, 20), - stop=st.integers(-20, 20), - step=st.integers(0, 7)) -def test_lsb0_slicing(length, start, stop, step): - if start == -20: - start = None - if stop == -20: - stop = None - if step == 0: - step = None - values_fwd = list(range(0, length)) - values_bwd = list(range(0, length)) - values_bwd.reverse() - - # Convert the start, stop, step to a range over the length - start1, stop1, step1 = slice(start, stop, step).indices(length) - values1 = values_fwd[start1:stop1:step1] - - lsb0key = offset_slice_indices_lsb0(slice(start, stop, step), length) - values2 = values_bwd[lsb0key.start:lsb0key.stop:lsb0key.step] - values2.reverse() - assert values1 == values2 + colour_escape = re.compile(r"(?:\x1B[@-_])[0-?]*[ -/]*[@-~]") + return colour_escape.sub("", s) + class TestCreation: + def test_explicit_factory_methods(self, tmp_path): + assert Bits.from_string("0xf, u4=1") == "0xf1" + assert Bits.from_dtype("u8", 12) == "0x0c" + assert Bits.from_bytes(b"\xf0", offset=1, length=3) == "0b111" + assert Bits.from_bools([True, 0, "x"]) == "0b101" + assert Bits.from_zeros(5) == "0b00000" + assert Bits.from_ones(5) == "0b11111" + assert Bits.from_joined(["0xa", "0xb"]) == "0xab" + assert Bits.from_joined([]) == Bits() + + filename = tmp_path / "factory.bin" + filename.write_bytes(b"\x12\x34") + assert Bits.from_file(filename) == "0x1234" + with filename.open("rb") as f: + assert Bits.from_file(f, offset=4, length=8) == "0x23" + + def test_to_bitarray(self): + bits = Bits("0b101") + bitarray = bits.to_bitarray() + assert type(bitarray) is BitArray + assert bitarray == bits + bitarray.append("0b1") + assert bits == "0b101" + assert bitarray == "0b1011" + + def test_tibs_conversion(self): + tibs = Tibs.from_bin("101") + bits = Bits.from_tibs(tibs) + assert type(bits) is Bits + assert bits == "0b101" + assert bits.to_tibs() is tibs + assert Bits("0b101").to_tibs() == tibs + + mutibs = Mutibs.from_bin("101") + bits = Bits.from_tibs(mutibs) + assert type(bits) is Bits + assert bits == "0b101" + mutibs.append(0) + assert bits == "0b101" + + @pytest.mark.parametrize("tibs_type", [Tibs, Mutibs]) + def test_constructor_accepts_tibs_types(self, tibs_type): + tibs = tibs_type.from_bin("101") + bits = Bits(tibs) + assert type(bits) is Bits + assert bits == "0b101" + if tibs_type is Tibs: + assert bits.to_tibs() is tibs + + @pytest.mark.parametrize("tibs_type", [Tibs, Mutibs]) + def test_constructor_rejects_tibs_types_with_length_or_offset(self, tibs_type): + tibs = tibs_type.from_bin("101") + with pytest.raises(bitstring.CreationError, match="explicit lengths or offsets"): + Bits(tibs, length=2) + with pytest.raises(bitstring.CreationError, match="explicit lengths or offsets"): + Bits(tibs, offset=1) + + def test_mutibs_input_is_copied(self): + source = Mutibs.from_bin("101") + constructed = Bits(source) + from_tibs = Bits.from_tibs(source) + + source.append(0) + + assert constructed == "0b101" + assert from_tibs == "0b101" + + def test_tibs_types_promote_as_bits_type(self): + tibs = Tibs.from_bin("101") + mutibs = Mutibs.from_bin("101") + bits = Bits("0b00101") + + assert bits.find(tibs) == 2 + assert bits.endswith(mutibs) + assert Bits("0b10100").startswith(tibs) + assert Bits.from_joined([Tibs.from_bin("10"), mutibs]) == "0b10101" + assert bitstring.pack("bits3", tibs) == "0b101" + assert bitstring.pack("bits3", mutibs) == "0b101" + + def test_to_bytes_and_to_file_aliases(self, tmp_path): + bits = Bits("0b101") + assert bits.to_bytes() == b"\xa0" + assert bits.tobytes() == b"\xa0" + assert bytes(bits) == b"\xa0" + + filename = tmp_path / "bits.bin" + with filename.open("wb") as f: + bits.to_file(f) + assert filename.read_bytes() == b"\xa0" + + filename = tmp_path / "bits_alias.bin" + with filename.open("wb") as f: + bits.tofile(f) + assert filename.read_bytes() == b"\xa0" + + @pytest.mark.parametrize("cls", [Bits, BitArray]) + def test_positional_integer_constructor_removed(self, cls): + with pytest.raises(TypeError, match="from_zeros"): + cls(5) + + @pytest.mark.parametrize("cls", [Bits, BitArray]) + def test_ambiguous_constructor_sources_removed(self, cls, tmp_path): + with pytest.raises(TypeError, match="from_bools"): + cls([1, 0, 1]) + with pytest.raises(TypeError, match="from_bytes"): + cls(io.BytesIO(b"\xff")) + with pytest.raises(TypeError, match="from_bytes"): + cls(array.array("B", [0xff])) + + filename = tmp_path / "source.bin" + filename.write_bytes(b"\xff") + with filename.open("rb") as f: + with pytest.raises(TypeError, match="from_file"): + cls(f) + + @pytest.mark.parametrize("cls", [Bits, BitArray]) + def test_fromstring_compatibility_alias(self, cls): + s = cls.fromstring("u4=15, 0b01") + assert type(s) is cls + assert s == cls.from_string("u4=15, 0b01") + def test_creation_from_bytes(self): - s = Bits(bytes=b'\xa0\xff') - assert (s.len, s.hex) == (16, 'a0ff') - s = Bits(bytes=b'abc', length=0) - assert s == '' + s = Bits(bytes=b"\xa0\xff") + assert (s.len, s.hex) == (16, "a0ff") + s = Bits(bytes=b"abc", length=0) + assert s == "" @given(st.binary()) def test_creation_from_bytes_roundtrip(self, data): @@ -61,90 +157,98 @@ def test_creation_from_bytes_roundtrip(self, data): def test_creation_from_bytes_errors(self): with pytest.raises(bitstring.CreationError): - Bits(bytes=b'abc', length=25) + Bits(bytes=b"abc", length=25) def test_creation_from_data_with_offset(self): - s1 = Bits(bytes=b'\x0b\x1c\x2f', offset=0, length=20) - s2 = Bits(bytes=b'\xa0\xb1\xC2', offset=4) - assert (s2.len, s2.hex) == (20, '0b1c2') - assert (s1.len, s1.hex) == (20, '0b1c2') + s1 = Bits(bytes=b"\x0b\x1c\x2f", offset=0, length=20) + s2 = Bits(bytes=b"\xa0\xb1\xc2", offset=4) + assert (s2.len, s2.hex) == (20, "0b1c2") + assert (s1.len, s1.hex) == (20, "0b1c2") assert s1 == s2 def test_creation_from_hex(self): - s = Bits(hex='0xA0ff') - assert (s.len, s.hex) == (16, 'a0ff') - s = Bits(hex='0x0x0X') - assert (s.length, s.hex) == (0, '') + s = Bits(hex="0xA0ff") + assert (s.len, s.hex) == (16, "a0ff") + s = Bits(hex="0x0x0X") + assert (s.length, s.hex) == (0, "") def test_creation_from_hex_with_whitespace(self): - s = Bits(hex=' \n0 X a 4e \r3 \n') - assert s.hex == 'a4e3' + s = Bits(hex=" \n0 X a 4e \r3 \n") + assert s.hex == "a4e3" - @pytest.mark.parametrize("bad_val", ['0xx0', '0xX0', '0Xx0', '-2e']) + @pytest.mark.parametrize("bad_val", ["0xx0", "0xX0", "0Xx0", "-2e"]) def test_creation_from_hex_errors(self, bad_val: str): with pytest.raises(bitstring.CreationError): Bits(hex=bad_val) with pytest.raises(bitstring.CreationError): - Bits('0x2', length=2) + Bits("0x2", length=2) with pytest.raises(bitstring.CreationError): - Bits('0x3', offset=1) + Bits("0x3", offset=1) def test_creation_from_bin(self): - s = Bits(bin='1010000011111111') - assert (s.length, s.hex) == (16, 'a0ff') - s = Bits(bin='00')[:1] - assert s.bin == '0' - s = Bits(bin=' 0000 \n 0001\r ') - assert s.bin == '00000001' + s = Bits(bin="1010000011111111") + assert (s.length, s.hex) == (16, "a0ff") + s = Bits(bin="00")[:1] + assert s.bin == "0" + s = Bits(bin=" 0000 \n 0001\r ") + assert s.bin == "00000001" def test_creation_from_bin_with_whitespace(self): - s = Bits(bin=' \r\r\n0 B 00 1 1 \t0 ') - assert s.bin == '00110' + s = Bits(bin=" \r\r\n0 B 00 1 1 \t0 ") + assert s.bin == "00110" def test_creation_from_oct_errors(self): - s = Bits('0b00011') + s = Bits("0b00011") with pytest.raises(bitstring.InterpretError): _ = s.oct with pytest.raises(bitstring.CreationError): - _ = Bits('oct=8') + _ = Bits("oct=8") - def test_creation_from_uint_with_offset(self): + def test_creation_from_u_with_offset(self): with pytest.raises(bitstring.CreationError): - Bits(uint=12, length=8, offset=1) + Bits(u=12, length=8, offset=1) + + def test_long_numeric_keyword_initialisers_are_compatibility_aliases(self): + assert Bits(uint=4, length=10) == Bits(u=4, length=10) + assert Bits(int=-2, length=10) == Bits(i=-2, length=10) + assert Bits(float=1.5, length=16) == Bits(f=1.5, length=16) + assert BitArray(uint=4, length=10) == BitArray(u=4, length=10) + assert BitArray(int=-2, length=10) == BitArray(i=-2, length=10) + assert BitArray(float=1.5, length=16) == BitArray(f=1.5, length=16) - def test_creation_from_uint_errors(self): + def test_creation_from_u_errors(self): with pytest.raises(bitstring.CreationError): - Bits(uint=-1, length=10) + Bits(u=-1, length=10) with pytest.raises(bitstring.CreationError): - Bits(uint=12) + Bits(u=12) with pytest.raises(bitstring.CreationError): - Bits(uint=4, length=2) + Bits(u=4, length=2) with pytest.raises(bitstring.CreationError): - Bits(uint=0, length=0) + Bits(u=0, length=0) with pytest.raises(bitstring.CreationError): - Bits(uint=12, length=-12) - - def test_creation_from_int(self): - s = Bits(int=0, length=4) - assert s.bin == '0000' - s = Bits(int=1, length=2) - assert s.bin == '01' - s = Bits(int=-1, length=11) - assert s.bin == '11111111111' - s = Bits(int=12, length=7) + Bits(u=12, length=-12) + + def test_creation_from_i(self): + s = Bits(i=0, length=4) + assert s.bin == "0000" + s = Bits(i=1, length=2) + assert s.bin == "01" + s = Bits(i=-1, length=11) + assert s.bin == "11111111111" + s = Bits(i=12, length=7) assert s.int == 12 - s = Bits(int=-243, length=108) + s = Bits(i=-243, length=108) assert (s.int, s.length) == (-243, 108) for length in range(6, 10): for value in range(-17, 17): - s = Bits(int=value, length=length) + s = Bits(i=value, length=length) assert (s.int, s.length) == (value, length) - _ = Bits(int=10, length=8) + _ = Bits(i=10, length=8) - @pytest.mark.parametrize("int_, length", [[-1, 0], [12, None], [4, 3], [-5, 3]]) - def test_creation_from_int_errors(self, int_, length): + @pytest.mark.parametrize("value, length", [[-1, 0], [12, None], [4, 3], [-5, 3]]) + def test_creation_from_i_errors(self, value, length): with pytest.raises(bitstring.CreationError): - _ = Bits(int=int_, length=length) + _ = Bits(i=value, length=length) def test_creation_from_se(self): for i in range(-100, 10): @@ -161,8 +265,8 @@ def test_creation_from_se_errors(self): with pytest.raises(bitstring.CreationError): Bits(se=-5, length=33) with pytest.raises(bitstring.CreationError): - Bits('se2=0') - s = Bits(bin='001000') + Bits("se2=0") + s = Bits(bin="001000") with pytest.raises(bitstring.InterpretError): _ = s.se @@ -179,23 +283,23 @@ def test_creation_from_ue_errors(self): Bits(ue=-1) with pytest.raises(bitstring.CreationError): Bits(ue=1, length=12) - s = Bits(bin='10') + s = Bits(bin="10") with pytest.raises(bitstring.InterpretError): _ = s.ue def test_creation_from_bool(self): - a = Bits('bool=1') - assert a == 'bool=1' - b = Bits('bool:1=0') + a = Bits("bool=1") + assert a == "bool=1" + b = Bits("bool:1=0") assert b == [0] - c = bitstring.pack('bool=1, 2*bool', 0, 1) - assert c == '0b101' - d = bitstring.pack('bool:1=1, 2*bool1', 1, 0) - assert d == '0b110' + c = bitstring.pack("bool=1, 2*bool", 0, 1) + assert c == "0b101" + d = bitstring.pack("bool:1=1, 2*bool1", 1, 0) + assert d == "0b110" def test_creation_from_bool_errors(self): with pytest.raises(ValueError): - _ = Bits('bool=3') + _ = Bits("bool=3") with pytest.raises(bitstring.CreationError): _ = Bits(bool=0, length=2) @@ -203,78 +307,72 @@ def test_creation_keyword_error(self): with pytest.raises(bitstring.CreationError): Bits(squirrel=5) - @pytest.mark.skipif(bool(os.environ.get('BITSTRING_USE_RUST_CORE')), reason="bitarray not supported with Rust backend") - def test_creation_from_bitarray(self): - ba = bitarray.bitarray('0010') - bs = Bits(ba) - assert bs.bin == '0010' - bs2 = Bits(bitarray=ba) - assert bs2.bin == '0010' - - @pytest.mark.skipif(bool(os.environ.get('BITSTRING_USE_RUST_CORE')), reason="bitarray not supported with Rust backend") - def test_creation_from_frozen_bitarray(self): - fba = bitarray.frozenbitarray('111100001') - ba = Bits(fba) - assert ba.bin == '111100001' - bs2 = Bits(bitarray=fba) - assert bs2.bin == '111100001' - bs3 = Bits(bitarray=fba, offset=4) - assert bs3.bin == '00001' - bs3 = Bits(bitarray=fba, offset=4, length=4) - assert bs3.bin == '0000' - - def test_creation_from_bitarray_errors(self): - ba = bitarray.bitarray('0101') - with pytest.raises(bitstring.CreationError): - _ = Bits(bitarray=ba, length=5) - with pytest.raises(bitstring.CreationError): - _ = Bits(bitarray=ba, offset=5) - with pytest.raises(bitstring.CreationError): - _ = Bits(ba, length=-1) - def test_creation_from_memoryview(self): x = bytes(bytearray(range(20))) m = memoryview(x[10:15]) b = Bits(m) - assert b.unpack('5*u8') == [10, 11, 12, 13, 14] + assert b.unpack("5*u8") == [10, 11, 12, 13, 14] class TestInitialisation: def test_empty_init(self): a = Bits() - assert a == '' + assert a == "" def test_no_pos(self): - a = Bits('0xabcdef') + a = Bits("0xabcdef") with pytest.raises(AttributeError): _ = a.pos def test_find(self): - a = Bits('0xabcd') - r = a.find('0xbc') - assert r[0] == 4 - r = a.find('0x23462346246', bytealigned=True) - assert not r + a = Bits("0xabcd") + assert a.find("0xab") == 0 + assert "0xab" in a + r = a.find("0xbc") + assert r == 4 + r = a.find("0x23462346246", bytealigned=True) + assert r is None def test_rfind(self): - a = Bits('0b11101010010010') - b = a.rfind('0b010') - assert b[0] == 11 + a = Bits("0b11101010010010") + b = a.rfind("0b010") + assert b == 11 + assert a.rfind("0b111") == 0 def test_find_all(self): - a = Bits('0b0010011') + a = Bits("0b0010011") b = list(a.findall([1])) assert b == [2, 5, 6] - t = BitArray('0b10') - tp = list(t.findall('0b1')) + t = BitArray("0b10") + tp = list(t.findall("0b1")) assert tp == [0] + @pytest.mark.parametrize( + "call", + [ + lambda s: s.find("0b1", 0), + lambda s: s.rfind("0b1", 0), + lambda s: s.findall("0b1", 0), + lambda s: s.cut(1, 0), + lambda s: s.split("0b1", 0), + lambda s: s.startswith("0b1", 0), + lambda s: s.endswith("0b1", 0), + lambda s: s.replace("0b1", "0b0", 0), + lambda s: s.reverse(0, 2), + lambda s: s.rol(1, 0, 2), + lambda s: s.ror(1, 0, 2), + ], + ) + def test_optional_range_arguments_are_keyword_only(self, call): + with pytest.raises(TypeError): + call(BitArray("0b1010")) + class TestCut: def test_cut(self): - s = Bits('0b000111'*10) + s = Bits("0b000111" * 10) for t in s.cut(6): - assert t.bin == '000111' + assert t.bin == "000111" class TestInterleavedExpGolomb: @@ -304,8 +402,8 @@ def test_interpretation(self): assert Bits(sie=x).sie == x def test_errors(self): - for f in ['sie=100, 0b1001', '0b00', 'uie=100, 0b1001']: - s = Bits.fromstring(f) + for f in ["sie=100, 0b1001", "0b00", "uie=100, 0b1001"]: + s = Bits.from_string(f) with pytest.raises(bitstring.InterpretError): _ = s.sie with pytest.raises(bitstring.InterpretError): @@ -316,42 +414,42 @@ def test_errors(self): class TestFileBased: def setup_method(self): - filename = os.path.join(THIS_DIR, 'smalltestfile') + filename = os.path.join(THIS_DIR, "smalltestfile") self.a = Bits(filename=filename) self.b = Bits(filename=filename, offset=16) self.c = Bits(filename=filename, offset=20, length=16) self.d = Bits(filename=filename, offset=20, length=4) def test_creation_with_offset(self): - assert str(self.a) == '0x0123456789abcdef' - assert str(self.b) == '0x456789abcdef' - assert str(self.c) == '0x5678' + assert str(self.a) == "0x0123456789abcdef" + assert str(self.b) == "0x456789abcdef" + assert str(self.c) == "0x5678" def test_bit_operators(self): x = self.b[4:20] - assert x == '0x5678' + assert x == "0x5678" assert (x & self.c).hex == self.c.hex - assert self.c ^ self.b[4:20] == Bits(16) + assert self.c ^ self.b[4:20] == Bits.from_zeros(16) assert self.a[23:36] | self.c[3:] == self.c[3:] y = x & self.b[4:20] assert y == self.c assert repr(y) == repr(self.c) def test_addition(self): - _ = self.d + '0x1' + _ = self.d + "0x1" x = self.a[20:24] + self.c[-4:] + self.c[8:12] - assert x == '0x587' + assert x == "0x587" x = self.b + x - assert x.h == '456789abcdef587' + assert x.hex == "456789abcdef587" x = BitArray(x) del x[12:24] - assert x == '0x456abcdef587' + assert x == "0x456abcdef587" class TestComparisons: def test_unorderable(self): - a = Bits(5) - b = Bits(5) + a = Bits.from_zeros(5) + b = Bits.from_zeros(5) with pytest.raises(TypeError): _ = a < b with pytest.raises(TypeError): @@ -363,382 +461,281 @@ def test_unorderable(self): class TestSubclassing: - def test_is_instance(self): class SubBits(bitstring.Bits): pass + a = SubBits() assert isinstance(a, SubBits) def test_class_type(self): class SubBits(bitstring.Bits): pass + assert SubBits().__class__ == SubBits class TestLongBoolConversion: - def test_long_bool(self): - a = Bits(1000) + a = Bits.from_zeros(1000) b = bool(a) assert b is True class TestPadToken: - def test_creation(self): - a = Bits.fromstring('pad:10') - assert a == Bits(10) - b = Bits('pad:0') + a = Bits.from_string("pad:10") + assert a == Bits.from_zeros(10) + b = Bits("pad:0") assert b == Bits() - c = Bits('0b11, pad:1, 0b111') - assert c == Bits('0b110111') + c = Bits("0b11, pad:1, 0b111") + assert c == Bits("0b110111") def test_pack(self): - s = bitstring.pack('0b11, pad:3, 0b1') - assert s.bin == '110001' - d = bitstring.pack('pad:c', c=12) - assert d == Bits(12) - e = bitstring.pack('0xf, uint12, pad:1, bin, pad4, 0b10', 0, '111') - assert e.bin == '11110000000000000111000010' + s = bitstring.pack("0b11, pad:3, 0b1") + assert s.bin == "110001" + d = bitstring.pack("pad:c", c=12) + assert d == Bits.from_zeros(12) + e = bitstring.pack("0xf, uint12, pad:1, bin, pad4, 0b10", 0, "111") + assert e.bin == "11110000000000000111000010" def test_unpack(self): - s = Bits('0b111000111') - x, y = s.unpack('3, pad:3, 3') - assert (x, y.u) == ('0b111', 7) - x, y = s.unpack('2, pad2, bin') - assert (x.u2, y) == (3, '00111') - x = s.unpack('pad:1, pad:2, pad:3') + s = Bits("0b111000111") + x, y = s.unpack("3, pad:3, 3") + assert (x, y.u) == ("0b111", 7) + x, y = s.unpack("2, pad2, bin") + assert (x.u2, y) == (3, "00111") + x = s.unpack("pad:1, pad:2, pad:3") assert x == [] def test_unpack_bug(self): - t = Bits('0o755, ue=12, int3=-1') - a, b = t.unpack('pad:9, ue, int3') + t = Bits("0o755, ue=12, i3=-1") + a, b = t.unpack("pad:9, ue, i3") assert (a, b) == (12, -1) class TestModifiedByAddingBug: - def test_adding(self): - a = Bits('0b0') - b = Bits('0b11') + a = Bits("0b0") + b = Bits("0b11") c = a + b - assert c == '0b011' - assert a == '0b0' - assert b == '0b11' + assert c == "0b011" + assert a == "0b0" + assert b == "0b11" def test_adding2(self): - a = Bits(100) - b = Bits(101) + a = Bits.from_zeros(100) + b = Bits.from_zeros(101) c = a + b - assert a == Bits(100) - assert b == Bits(101) - assert c == Bits(201) + assert a == Bits.from_zeros(100) + assert b == Bits.from_zeros(101) + assert c == Bits.from_zeros(201) class TestWrongTypeBug: - def test_append_to_bits(self): a = Bits(BitArray()) with pytest.raises(AttributeError): - a.append('0b1') + a.append("0b1") assert type(a) == Bits - b = bitstring.ConstBitStream(bitstring.BitStream()) - assert type(b) == bitstring.ConstBitStream + b = Bits(BitArray()) + assert type(b) == Bits class TestInitFromArray: - - @given(st.sampled_from(['B', 'H', 'I', 'L', 'Q', 'f', 'd'])) + @given(st.sampled_from(["B", "H", "I", "L", "Q", "f", "d"])) def test_empty_array(self, t): a = array.array(t) - b = Bits(a) + b = Bits.from_bytes(a.tobytes()) assert b.length == 0 def test_single_byte(self): - a = array.array('B', b'\xff') - b = Bits(a) + a = array.array("B", b"\xff") + b = Bits.from_bytes(a.tobytes()) assert b.length == 8 - assert b.hex == 'ff' + assert b.hex == "ff" def test_signed_short(self): - a = array.array('h') + a = array.array("h") a.append(10) a.append(-1) - b = Bits(a) + b = Bits.from_bytes(a.tobytes()) assert b.length == 32 assert b.bytes == a.tobytes() def test_double(self): - a = array.array('d', [0.0, 1.0, 2.5]) - b = Bits(a) + a = array.array("d", [0.0, 1.0, 2.5]) + b = Bits.from_bytes(a.tobytes()) assert b.length == 192 - c, d, e = b.unpack('3*floatne:64') + c, d, e = b.unpack("3*fne:64") assert (c, d, e) == (0.0, 1.0, 2.5) class TestIteration: - def test_iterate_empty_bits(self): - assert list(Bits([])) == [] - assert list(Bits([1, 0])[1:1]) == [] + assert list(Bits.from_bools([])) == [] + assert list(Bits.from_bools([1, 0])[1:1]) == [] def test_iterate_non_empty_bits(self): - assert list(Bits([1, 0])) == [True, False] - assert list(Bits([1, 0, 0, 1])[1:3]) == [False, False] + assert list(Bits.from_bools([1, 0])) == [True, False] + assert list(Bits.from_bools([1, 0, 0, 1])[1:3]) == [False, False] def test_iterate_long_bits(self): - assert list(Bits([1, 0]) * 1024) == \ - [True, False] * 1024 + assert list(Bits.from_bools([1, 0]) * 1024) == [True, False] * 1024 - -class TestContainsBug: +class TestContainsBug: def test_contains(self): - a = Bits('0b1, 0x0001dead0001') - assert '0xdead' in a - assert not '0xfeed' in a + a = Bits("0b1, 0x0001dead0001") + assert "0xdead" in a + assert "0xfeed" not in a - assert '0b1' in Bits('0xf') - assert not '0b0' in Bits('0xf') + assert "0b1" in Bits("0xf") + assert "0b0" not in Bits("0xf") class TestByteStoreImmutablity: - def test_immutability_bug_append(self): - a = Bits('0b111') - b = a + '0b000' + a = Bits("0b111") + b = a + "0b000" c = BitArray(b) c[1] = 0 - assert c.bin == '101000' - assert a.b3 == '111' - assert b.bin == '111000' + assert c.bin == "101000" + assert a.bin3 == "111" + assert b.bin == "111000" def test_immutability_bug_prepend(self): - a = Bits('0b111') - b = '0b000' + a + a = Bits("0b111") + b = "0b000" + a c = BitArray(b) c[1] = 1 - assert b.bin == '000111' - assert c.bin == '010111' - - -class TestLsb0Indexing: - - @classmethod - def setup_class(cls): - bitstring.lsb0 = True - - @classmethod - def teardown_class(cls): - bitstring.lsb0 = False - - def test_get_single_bit(self): - a = Bits('0b000001111') - assert a[0] is True - assert a[3] is True - assert a[4] is False - assert a[8] is False - with pytest.raises(IndexError): - _ = a[9] - assert a[-1] is False - assert a[-5] is False - assert a[-6] is True - assert a[-9] is True - with pytest.raises(IndexError): - _ = a[-10] - - def test_simple_slicing(self): - a = Bits('0xabcdef') - assert a[0:4] == '0xf' - assert a[4:8] == '0xe' - assert a[:] == '0xabcdef' - assert a[4:] == '0xabcde' - assert a[-4:] == '0xa' - assert a[-8:-4] == '0xb' - assert a[:-8] == '0xcdef' - - def test_extended_slicing(self): - a = Bits('0b0100000100100100') - assert a[2::3] == '0b10111' - - def test_all(self): - a = Bits('0b000111') - assert a.all(1, [0, 1, 2]) - assert a.all(0, [3, 4, 5]) - - def test_any(self): - a = Bits('0b00000110') - assert a.any(1, [0, 1]) - assert a.any(0, [5, 6]) - - def test_startswith(self): - a = Bits('0b0000000111') - assert a.startswith('0b111') - assert not a.startswith('0b0') - assert a.startswith('0b011', start=1) - assert not a.startswith('0b0111', end=3) - assert a.startswith('0b0111', end=4) - - def test_ends_with(self): - a = Bits('0x1234abcd') - assert a.endswith('0x123') - assert not a.endswith('0xabcd') - - def test_lsb0_slicing_error(self): - a = Bits('0b01') - b = a[::-1] - assert b == '0b10' - t = Bits('0xf0a')[::-1] - assert t == '0x50f' - s = Bits('0xf0a')[::-1][::-1] - assert s == '0xf0a' - - -class TestLsb0Interpretations: - - @classmethod - def setup_class(cls): - bitstring.lsb0 = True - - @classmethod - def teardown_class(cls): - bitstring.lsb0 = False - - def test_uint(self): - a = Bits('0x01') - assert a == '0b00000001' - assert a.uint == 1 - assert a[0] is True - - def test_float(self): - a = Bits(float=0.25, length=32) - try: - bitstring.lsb0 = False - b = Bits(float=0.25, length=32) - finally: - bitstring.lsb0 = True - assert a.float == 0.25 - assert b.float == 0.25 - assert a.bin == b.bin - - def test_golomb(self): - with pytest.raises(bitstring.CreationError): - _ = Bits(ue=2) - with pytest.raises(bitstring.CreationError): - _ = Bits(se=2) - with pytest.raises(bitstring.CreationError): - _ = Bits(uie=2) - with pytest.raises(bitstring.CreationError): - _ = Bits(sie=2) - - def test_bytes(self): - a = Bits.fromstring('0xabcdef') - b = a.bytes - assert b == b'\xab\xcd\xef' - b = a.bytes3 - assert b == b'\xab\xcd\xef' + assert b.bin == "000111" + assert c.bin == "010111" class TestUnderscoresInLiterals: - def test_hex_creation(self): - a = Bits(hex='ab_cd__ef') - assert a.hex == 'abcdef' - b = Bits('0x0102_0304') + a = Bits(hex="ab_cd__ef") + assert a.hex == "abcdef" + b = Bits("0x0102_0304") assert b.uint == 0x0102_0304 def test_binary_creation(self): - a = Bits(bin='0000_0001_0010') - assert a.bin == '000000010010' - b = Bits.fromstring('0b0011_1100_1111_0000') - assert b.bin == '0011110011110000' + a = Bits(bin="0000_0001_0010") + assert a.bin == "000000010010" + b = Bits.from_string("0b0011_1100_1111_0000") + assert b.bin == "0011110011110000" v = 0b1010_0000 - c = Bits(uint=0b1010_0000, length=8) + c = Bits(u=0b1010_0000, length=8) assert c.uint == v def test_octal_creation(self): - a = Bits(oct='0011_2233_4455_6677') + a = Bits(oct="0011_2233_4455_6677") assert a.uint == 0o001122334455_6677 - b = Bits('0o123_321_123_321') + b = Bits("0o123_321_123_321") assert b.uint == 0o123_321_123321 class TestPrettyPrinting: - def test_simplest_cases(self): - a = Bits('0b101011110000') + a = Bits("0b101011110000") s = io.StringIO() a.pp(stream=s) - assert remove_unprintable(s.getvalue()) == """ [ + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 10101111 0000 ] """ + ) s = io.StringIO() - a.pp('hex', stream=s) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp("hex", stream=s) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: af 0 ] """ + ) s = io.StringIO() - a.pp('oct', stream=s) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp("oct", stream=s) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 5360 ] """ + ) def test_small_width(self): - a = Bits(20) + a = Bits.from_zeros(20) s = io.StringIO() - a.pp(fmt='b', stream=s, width=5) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp(fmt="bin", stream=s, width=5) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 00000000 8: 00000000 16: 0000 ] """ + ) def test_separator(self): - a = Bits('0x0f0f')*9 + a = Bits("0x0f0f") * 9 s = io.StringIO() - a.pp('hex:32', sep='!-!', stream=s) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp("hex:32", sep="!-!", stream=s) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 0f0f0f0f!-!0f0f0f0f!-!0f0f0f0f!-!0f0f0f0f ] + trailing_bits = 0x0f0f """ + ) def test_multi_line(self): - a = Bits(100) + a = Bits.from_zeros(100) s = io.StringIO() - a.pp('bin', sep='', stream=s, width=80) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp("bin", sep="", stream=s, width=80) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 000000000000000000000000000000000000000000000000000000000000000000000000 72: 0000000000000000000000000000 ] """ + ) def test_multiformat(self): - a = Bits('0b1111000011110000') + a = Bits("0b1111000011110000") s = io.StringIO() - a.pp(stream=s, fmt='bin, hex') - assert remove_unprintable(s.getvalue()) == """ [ + a.pp(stream=s, fmt="bin, hex") + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 11110000 11110000 : f0 f0 ] """ + ) s = io.StringIO() - a.pp(stream=s, fmt='hex, bin:12') - assert remove_unprintable(s.getvalue()) == """ [ + a.pp(stream=s, fmt="hex, bin:12") + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: f0f : 111100001111 ] + trailing_bits = 0x0 """ + ) def test_multi_line_multi_format(self): - a = Bits(int=-1, length=112) + a = Bits(i=-1, length=112) s = io.StringIO() - a.pp(stream=s, fmt='bin:8, hex:8', width=42) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp(stream=s, fmt="bin:8, hex:8", width=42) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 11111111 11111111 11111111 : ff ff ff 24: 11111111 11111111 11111111 : ff ff ff 48: 11111111 11111111 11111111 : ff ff ff @@ -746,9 +743,12 @@ def test_multi_line_multi_format(self): 96: 11111111 11111111 : ff ff ] """ + ) s = io.StringIO() - a.pp(stream=s, fmt='bin, hex', width=41) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp(stream=s, fmt="bin, hex", width=41) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 11111111 11111111 : ff ff 16: 11111111 11111111 : ff ff 32: 11111111 11111111 : ff ff @@ -758,29 +758,33 @@ def test_multi_line_multi_format(self): 96: 11111111 11111111 : ff ff ] """ + ) a = bytearray(range(0, 256)) b = Bits(bytes=a) s = io.StringIO() - b.pp(stream=s, fmt='bytes') - assert remove_unprintable(s.getvalue()) == r""" [ + b.pp(stream=s, fmt="bytes") + assert ( + remove_unprintable(s.getvalue()) + == r""" [ 0: ĀāĂă ĄąĆć ĈĉĊċ ČčĎď ĐđĒē ĔĕĖė ĘęĚě ĜĝĞğ !"# $%&' ()*+ ,-./ 0123 4567 89:; <=>? @ABC DEFG HIJK LMNO PQRS TUVW XYZ[ 736: \]^_ `abc defg hijk lmno pqrs tuvw xyz{ |}~ſ ƀƁƂƃ ƄƅƆƇ ƈƉƊƋ ƌƍƎƏ ƐƑƒƓ ƔƕƖƗ Ƙƙƚƛ ƜƝƞƟ ƠơƢƣ ƤƥƦƧ ƨƩƪƫ ƬƭƮƯ ưƱƲƳ ƴƵƶƷ 1472: Ƹƹƺƻ Ƽƽƾƿ ǀǁǂǃ DŽDždžLJ LjljNJNj njǍǎǏ ǐǑǒǓ ǔǕǖǗ ǘǙǚǛ ǜǝǞǟ ǠǡǢǣ ǤǥǦǧ ǨǩǪǫ ǬǭǮǯ ǰDZDzdz ǴǵǶǷ ǸǹǺǻ ǼǽǾÿ ] """ + ) def test_group_size_errors(self): - a = Bits(120) + a = Bits.from_zeros(120) with pytest.raises(ValueError): - a.pp('hex:3') + a.pp("hex:3") with pytest.raises(ValueError): - a.pp('hex:4, oct') + a.pp("hex:4, oct") def test_zero_group_size(self): - a = Bits(600) + a = Bits.from_zeros(600) s = io.StringIO() - a.pp('b0', stream=s, show_offset=False) + a.pp("bin0", stream=s, show_offset=False) expected_output = """ [ 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 @@ -791,9 +795,9 @@ def test_zero_group_size(self): """ assert remove_unprintable(s.getvalue()) == expected_output - a = Bits(400) + a = Bits.from_zeros(400) s = io.StringIO() - a.pp(stream=s, fmt='hex:0', show_offset=False, width=80) + a.pp(stream=s, fmt="hex:0", show_offset=False, width=80) expected_output = """ [ 00000000000000000000000000000000000000000000000000000000000000000000000000000000 00000000000000000000 @@ -802,8 +806,8 @@ def test_zero_group_size(self): assert remove_unprintable(s.getvalue()) == expected_output s = io.StringIO() - a = Bits(uint=10, length=48) - a.pp(stream=s, width=20, fmt='hex:0, oct:0', show_offset=False) + a = Bits(u=10, length=48) + a.pp(stream=s, width=20, fmt="hex:0, oct:0", show_offset=False) expected_output = """ [ 000000 : 00000000 00000a : 00000012 @@ -812,9 +816,9 @@ def test_zero_group_size(self): assert remove_unprintable(s.getvalue()) == expected_output def test_oct(self): - a = Bits('0o01234567'*20) + a = Bits("0o01234567" * 20) s = io.StringIO() - a.pp(stream=s, fmt='o', show_offset=False, width=20) + a.pp(stream=s, fmt="oct", show_offset=False, width=20) expected_output = """ [ 0123 4567 0123 4567 0123 4567 0123 4567 @@ -831,7 +835,7 @@ def test_oct(self): assert remove_unprintable(s.getvalue()) == expected_output t = io.StringIO() - a.pp('h, oct:0', width=1, show_offset=False, stream=t) + a.pp("hex, oct:0", width=1, show_offset=False, stream=t) expected_output = """ [ 053977 : 01234567 053977 : 01234567 @@ -858,31 +862,28 @@ def test_oct(self): assert remove_unprintable(t.getvalue()) == expected_output def test_bytes(self): - a = Bits(bytes=b'helloworld!!'*5) + a = Bits(bytes=b"helloworld!!" * 5) s = io.StringIO() - a.pp(stream=s, fmt='bytes', show_offset=False, width=48) - expected_output = ( -""" [ + a.pp(stream=s, fmt="bytes", show_offset=False, width=48) + expected_output = """ [ hell owor ld!! hell owor ld!! hell owor ld!! hell owor ld!! hell owor ld!! ] -""") +""" assert remove_unprintable(s.getvalue()) == expected_output s = io.StringIO() - a.pp(stream=s, fmt='bytes0', show_offset=False, width=40) - expected_output = ( -""" [ + a.pp(stream=s, fmt="bytes0", show_offset=False, width=40) + expected_output = """ [ helloworld!!helloworld!!helloworld!!hell oworld!!helloworld!! ] """ - ) assert remove_unprintable(s.getvalue()) == expected_output def test_bool(self): - a = Bits('0b1100') + a = Bits("0b1100") s = io.StringIO() - a.pp(stream=s, fmt='bool', show_offset=False, width=20) + a.pp(stream=s, fmt="bool", show_offset=False, width=20) expected_output = """ [ 1 1 0 0 ] @@ -891,65 +892,52 @@ def test_bool(self): class TestPrettyPrintingErrors: - def test_wrong_formats(self): - a = Bits('0x12341234') + a = Bits("0x12341234") with pytest.raises(ValueError): - a.pp('binary') + a.pp("binary") with pytest.raises(ValueError): - a.pp('bin, bin, bin') + a.pp("bin, bin, bin") def test_interpret_problems(self): - a = Bits(7) + a = Bits.from_zeros(7) with pytest.raises(InterpretError): - a.pp('oct') + a.pp("oct") with pytest.raises(InterpretError): - a.pp('hex') + a.pp("hex") with pytest.raises(InterpretError): - a.pp('bin, bytes') - - -class TestPrettyPrinting_LSB0: - - def setup_method(self) -> None: - bitstring.lsb0 = True + a.pp("bin, bytes") - def teardown_method(self) -> None: - bitstring.lsb0 = False - - def test_bin(self): - a = Bits(bin='1111 0000 0000 1111 1010') - s = io.StringIO() - a.pp('bin', stream=s, width=5) - assert remove_unprintable(s.getvalue()) == """ [ -11111010 :0 -00000000 :8 - 1111 :16 -] -""" class TestPrettyPrinting_NewFormats: - - def test_float(self): - a = Bits('float32=10.5') + def test_f_formats(self): + a = Bits("f32=10.5") s = io.StringIO() - a.pp('float32', stream=s) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp("f32", stream=s) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 10.5 ] """ + ) s = io.StringIO() - a.pp('float16', stream=s) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp("f16", stream=s) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 2.578125 0.0 ] """ + ) - def test_uint(self): - a = Bits().join([Bits(uint=x, length=12) for x in range(40, 105)]) + def test_u(self): + a = Bits().join([Bits(u=x, length=12) for x in range(40, 105)]) s = io.StringIO() - a.pp('uint, h12', stream=s) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp("u, hex12", stream=s) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 40 41 42 43 44 45 46 47 48 49 50 51 : 028 029 02a 02b 02c 02d 02e 02f 030 031 032 033 144: 52 53 54 55 56 57 58 59 60 61 62 63 : 034 035 036 037 038 039 03a 03b 03c 03d 03e 03f 288: 64 65 66 67 68 69 70 71 72 73 74 75 : 040 041 042 043 044 045 046 047 048 049 04a 04b @@ -958,51 +946,62 @@ def test_uint(self): 720: 100 101 102 103 104 : 064 065 066 067 068 ] """ + ) - def test_float(self): - a = BitArray(float=76.25, length=64) + '0b11111' + def test_i_and_f(self): + a = BitArray(f=76.25, length=64) + "0b11111" s = io.StringIO() - a.pp('i64, float', stream=s) - assert remove_unprintable(s.getvalue()) == """ [ + a.pp("i64, f", stream=s) + assert ( + remove_unprintable(s.getvalue()) + == """ [ 0: 4635066033680416768 : 76.25 ] + trailing_bits = 0b11111 """ + ) -class TestCopy: +class TestCopy: def test_copy_method(self): - s = Bits('0xc00dee') + s = Bits("0xc00dee") t = s.copy() assert s == t class TestNativeEndianIntegers: - - def test_uintne(self): - s = Bits(uintne=454, length=160) - t = Bits('uintne160=454') + def test_une(self): + s = Bits(une=454, length=160) + t = Bits("une160=454") assert s == t - def test_intne(self): - s = Bits(intne=-1000, length=64) - t = Bits('intne:64=-1000') + def test_ine(self): + s = Bits(ine=-1000, length=64) + t = Bits("ine:64=-1000") assert s == t class TestNonNativeEndianIntegers: - def setup_method(self) -> None: - bitstring.byteorder = 'little' if bitstring.byteorder == 'big' else 'little' + bitstring.byteorder = "little" if bitstring.byteorder == "big" else "little" def teardown_method(self) -> None: self.setup_method() - def test_uintne(self): - s = Bits(uintne=454, length=160) - t = Bits('uintne160=454') + def test_une(self): + s = Bits(une=454, length=160) + t = Bits("une160=454") assert s == t - def test_intne(self): - s = Bits(intne=-1000, length=64) - t = Bits('intne:64=-1000') + def test_ine(self): + s = Bits(ine=-1000, length=64) + t = Bits("ine:64=-1000") assert s == t + + +def test_large_ints(): + s = Bits(i=-1, length=123) + assert s.int == -1 + s = Bits(i=-1, length=201) + assert s.int == -1 + s = Bits(u=12, length=201) + assert s.uint == 12 diff --git a/tests/test_bitstore.py b/tests/test_bitstore.py index d3f401d8..02281a08 100644 --- a/tests/test_bitstore.py +++ b/tests/test_bitstore.py @@ -6,7 +6,6 @@ import bitstring import sys MutableBitStore = bitstring.bitstore.MutableBitStore -offset_slice_indices_lsb0 = bitstring.helpers.offset_slice_indices_lsb0 sys.path.insert(0, '..') @@ -27,69 +26,3 @@ def test_getting_int(self): _ = a.getindex(3) with pytest.raises(IndexError): _ = a.getindex(-4) - - -class TestBasicLSB0Functionality: - - @classmethod - def setup_class(cls): - bitstring.lsb0 = True - - @classmethod - def teardown_class(cls): - bitstring.lsb0 = False - - def test_getting_int(self): - a = MutableBitStore.from_bin('001') - assert a.getindex(0) == 1 - assert a.getindex(1) == 0 - assert a.getindex(2) == 0 - - assert a.getindex(-1) == 0 - assert a.getindex(-2) == 0 - assert a.getindex(-3) == 1 - - with pytest.raises(IndexError): - _ = a.getindex(3) - with pytest.raises(IndexError): - _ = a.getindex(-4) - - def test_getting_slice(self): - a = MutableBitStore.from_bytes(b'12345678') - assert a.getslice(None, None).to_bytes() == b'12345678' - assert a.getslice(None, -8).to_bytes() == b'2345678' - assert a.getslice(8, None).to_bytes() == b'1234567' - assert a.getslice(16, 24).to_bytes() == b'6' - - def test_setting_int(self): - a = MutableBitStore.from_zeros(5) - a[0] = 1 - assert a.to_bin() == '00001' - a[-1] = 1 - assert a.to_bin() == '10001' - with pytest.raises(IndexError): - a[5] = 1 - with pytest.raises(IndexError): - a[-6] = 0 - - -class TestGettingSlices: - - def teardown_method(self) -> None: - bitstring.lsb0 = False - - def test_everything(self): - a = MutableBitStore.from_bin('010010001000110111001111101101001111') - - # Try combination of start and stop for msb0 and get the result. - # Convert to start and stop needed for lsb0 - options = [5, 2, -2, 1, 7, -3, -9, 0, -1, -len(a), len(a), len(a) - 1, -len(a) - 1, -100, 100, None] - for start_option in options: - for end_option in options: - bitstring.lsb0 = True - lsb0 = a.getslice(start_option, end_option) - bitstring.lsb0 = False - msb0 = a.getslice(start_option, end_option) - new_slice = offset_slice_indices_lsb0(slice(start_option, end_option, None), len(a)) - new_start, new_end = new_slice.start, new_slice.stop - assert len(msb0) == len(lsb0), f"[{start_option}: {end_option}] -> [{new_start}: {new_end}] len(msb0)={len(msb0)}, len(lsb0)={len(lsb0)}" diff --git a/tests/test_bitstream.py b/tests/test_bitstream.py deleted file mode 100644 index ddd8f47f..00000000 --- a/tests/test_bitstream.py +++ /dev/null @@ -1,4210 +0,0 @@ -#!/usr/bin/env python - -import pytest -import sys -sys.path.insert(0, '..') -import bitstring -import copy -import os -import collections -from bitstring import Bits, BitStream, ConstBitStream, pack, Dtype, BitArray - -THIS_DIR = os.path.dirname(os.path.abspath(__file__)) - - -class TestFlexibleInitialisation: - def test_flexible_initialisation(self): - a = BitStream('uint:8=12') - c = BitStream(' uint : 8 = 12') - assert a == c == BitStream(uint=12, length=8) - assert a.uint == 12 - a = BitStream(' int:2= -1') - b = BitStream('int :2 = -1') - c = BitStream(' int: 2 =-1 ') - assert a == b == c == BitStream(int=-1, length=2) - - def test_flexible_initialisation2(self): - h = BitStream('hex=12') - o = BitStream('oct=33') - b = BitStream('bin=10') - assert h == '0x12' - assert o == '0o33' - assert b == '0b10' - - def test_flexible_initialisation3(self): - for s in ['se=-1', ' se = -1 ', 'se = -1']: - a = BitStream(s) - assert a.se == -1 - for s in ['ue=23', 'ue =23', 'ue = 23']: - a = BitStream(s) - assert a.ue == 23 - - def test_multiple_string_initialisation(self): - a = BitStream('0b1 , 0x1') - assert a == '0b10001' - a = BitStream('ue=5, ue=1, se=-2') - assert a.read('ue') == 5 - assert a.read('ue') == 1 - assert a.read('se') == -2 - b = BitStream('uint:32 = 12, 0b11') + 'int:100=-100, 0o44' - assert b.read(32).uint == 12 - assert b.read(2).bin == '11' - assert b.read(100).int == -100 - - -class TestReading: - def test_read_bits(self): - s = BitStream(bytes=b'\x4d\x55') - assert s.read(4).hex == '4' - assert s.read(8).hex == 'd5' - assert s.read(1) == [0] - assert s.read(3).bin == '101' - assert not s.read(0) - - def test_read_byte(self): - s = BitStream(hex='4d55') - assert s.read(8).hex == '4d' - assert s.read(8).hex == '55' - - def test_read_bytes(self): - s = BitStream(hex='0x112233448811') - assert s.read(3 * 8).hex == '112233' - with pytest.raises(ValueError): - s.read(-16) - s.bitpos += 1 - assert s.read(2 * 8).bin == '1000100100010000' - - def test_read_ue(self): - with pytest.raises(bitstring.InterpretError): - _ = BitStream('').ue - # The numbers 0 to 8 as unsigned Exponential-Golomb codes - s = BitStream(bin='1 010 011 00100 00101 00110 00111 0001000 0001001') - assert s.pos == 0 - for i in range(9): - assert s.read('ue') == i - with pytest.raises(bitstring.ReadError): - s.read('ue') - - def test_read_se(self): - s = BitStream(bin='010 00110 0001010 0001000 00111') - assert s.read('se') == 1 - assert s.read('se') == 3 - assert s.readlist(3 * ['se']) == [5, 4, -3] - - -class TestFind: - def test_find1(self): - s = ConstBitStream(bin='0b0000110110000') - assert s.find(BitStream(bin='11011')) - assert s.bitpos == 4 - assert s.read(5).bin == '11011' - s.bitpos = 0 - assert not s.find('0b11001', False) - - def test_find2(self): - s = BitStream(bin='0') - assert s.find(s, False) - assert s.pos == 0 - assert not s.find('0b00', False) - with pytest.raises(ValueError): - s.find(BitStream()) - - def test_find_with_offset(self): - s = BitStream(hex='0x112233')[4:] - assert s.find('0x23', False) - assert s.pos == 8 - - def test_find_corner_cases(self): - s = BitStream(bin='000111000111') - assert s.find('0b000') - assert s.pos == 0 - assert s.find('0b000') - assert s.pos == 0 - assert s.find('0b0111000111') - assert s.pos == 2 - assert s.find('0b000', start=2) - assert s.pos == 6 - assert s.find('0b111', start=6) - assert s.pos == 9 - s.pos += 2 - assert s.find('0b1', start=s.pos) - - def test_find_bytes(self): - s = BitStream.fromstring('0x010203040102ff') - assert s.find('0x05', bytealigned=True) ==() - assert s.find('0x02', bytealigned=True) == (8,) - assert s.read(16).hex == '0203' - assert s.find('0x02', start=s.bitpos, bytealigned=True) == (40,) - s.read(1) - assert not s.find('0x02', start=s.bitpos, bytealigned=True) - - def test_find_bytes_aligned_corner_cases(self): - s = BitStream('0xff') - assert s.find(s) - assert not s.find(BitStream(hex='0x12')) - assert not s.find(BitStream(hex='0xffff')) - - def test_find_bytes_bitpos(self): - s = BitStream(hex='0x1122334455') - s.pos = 2 - s.find('0x66', bytealigned=True) - assert s.pos == 2 - s.pos = 38 - s.find('0x66', bytealigned=True) - assert s.pos == 38 - - def test_find_byte_aligned(self): - s = BitStream(hex='0x12345678') - assert s.find(BitStream(hex='0x56'), bytealigned=True) - assert s.bytepos == 2 - s.pos = 0 - assert not s.find(BitStream(hex='0x45'), bytealigned=True) - s = BitStream('0x1234') - s.find('0x1234') - assert s.find('0x1234') - s += '0b111' - s.pos = 3 - s.find('0b1', start=17, bytealigned=True) - assert not s.find('0b1', start=17, bytealigned=True) - assert s.pos == 3 - - def test_find_byte_aligned_with_offset(self): - s = BitStream(hex='0x112233')[4:] - assert s.find(BitStream(hex='0x23')) - - def test_find_byte_aligned_errors(self): - s = BitStream(hex='0xffff') - with pytest.raises(ValueError): - s.find('') - with pytest.raises(ValueError): - s.find(BitStream()) - - -class TestRfind: - def test_rfind(self): - a = BitStream('0b001001001') - b = a.rfind('0b001') - assert b == (6,) - assert a.pos == 6 - big = BitStream(length=100000) + '0x12' + BitStream(length=10000) - found = big.rfind('0x12', bytealigned=True) - assert found == (100000,) - assert big.pos == 100000 - - def test_rfind_byte_aligned(self): - a = BitStream('0x8888') - b = a.rfind('0b1', bytealigned=True) - assert b == (8,) - assert a.pos == 8 - - def test_rfind_startbit(self): - a = BitStream('0x0000ffffff') - b = a.rfind('0x0000', start=1, bytealigned=True) - assert b == () - assert a.pos == 0 - b = a.rfind('0x00', start=1, bytealigned=True) - assert b == (8,) - assert a.pos == 8 - - def test_rfind_endbit(self): - a = BitStream('0x000fff') - b = a.rfind('0b011', start=0, end=14, bytealigned=False) - assert bool(b) == True - b = a.rfind('0b011', 0, 13, False) - assert b == () - - def test_rfind_errors(self): - a = BitStream('0x43234234') - with pytest.raises(ValueError): - a.rfind('', bytealigned=True) - with pytest.raises(ValueError): - a.rfind('0b1', start=-99, bytealigned=True) - with pytest.raises(ValueError): - a.rfind('0b1', end=33, bytealigned=True) - with pytest.raises(ValueError): - a.rfind('0b1', start=10, end=9, bytealigned=True) - - -class TestShift: - def test_shift_left(self): - s = BitStream.fromstring('0b1010') - t = s << 1 - assert s.bin == '1010' - assert t.bin == '0100' - s = t << 0 - assert s == '0b0100' - t = s << 100 - assert t.bin == '0000' - - def test_shift_left_errors(self): - s = BitStream() - with pytest.raises(ValueError): - s << 1 - s = BitStream('0xf') - with pytest.raises(ValueError): - s << -1 - - def test_shift_right(self): - s = BitStream('0b1010') - t = s >> 1 - assert s.bin == '1010' - assert t.bin == '0101' - q = s >> 0 - assert q == '0b1010' - q.replace('0b1010', '') - t = s >> 100 - assert t.bin == '0000' - - def test_shift_right_errors(self): - s = BitStream() - with pytest.raises(ValueError): - s >> 1 - s = BitStream('0xf') - with pytest.raises(ValueError): - s >> -1 - - def test_shift_right_in_place(self): - s = BitArray.fromstring('0xffff') - s = s[4:12] - s >>= 1 - assert s == '0b01111111' - - s = BitStream.fromstring('0xffff') - s = s[4:12] - s >>= 1 - assert s == '0b01111111' - s = BitStream('0b11011') - s >>= 2 - assert s.bin == '00110' - s >>= 100000000000000 - assert s.bin == '00000' - s = BitStream('0xff') - s >>= 1 - assert s == '0x7f' - s >>= 0 - assert s == '0x7f' - - def test_shift_in_place_whole_bitstring(self): - s = BitStream('0xabcd') - s >>= len(s) - assert s == '0x0000' - - def test_shift_right_in_place_errors(self): - s = BitStream() - with pytest.raises(ValueError): - s >>= 1 - s += '0b11' - with pytest.raises(ValueError): - s >>= -1 - - def test_shift_left_in_place(self): - s = BitStream('0xffff') - t = s[4:12] - t <<= 2 - assert t == '0b11111100' - s = BitStream('0b11011') - s <<= 2 - assert s.bin == '01100' - s <<= 100000000000000000000 - assert s.bin == '00000' - s = BitStream('0xff') - s <<= 1 - assert s == '0xfe' - s <<= 0 - assert s == '0xfe' - - def test_shift_left_in_place_errors(self): - s = BitStream() - with pytest.raises(ValueError): - s <<= 1 - s += '0b11' - with pytest.raises(ValueError): - s <<= -1 - - -class TestReplace: - def test_replace1(self): - a = BitStream('0b1') - n = a.replace('0b1', '0b0', bytealigned=True) - assert a.bin == '0' - assert n == 1 - n = a.replace('0b1', '0b0', bytealigned=True) - assert n == 0 - - def test_replace2(self): - a = BitStream('0b00001111111') - n = a.replace('0b1', '0b0', bytealigned=True) - assert a.bin == '00001111011' - assert n == 1 - n = a.replace('0b1', '0b0', bytealigned=False) - assert a.bin == '00000000000' - assert n == 6 - - def test_replace3(self): - a = BitStream('0b0') - n = a.replace('0b0', '0b110011111', bytealigned=True) - assert n == 1 - assert a.bin == '110011111' - n = a.replace('0b11', '', bytealigned=False) - assert n == 3 - assert a.bin == '001' - - def test_replace4(self): - a = BitStream('0x00114723ef4732344700') - n = a.replace('0x47', '0x00', bytealigned=True) - assert n == 3 - assert a.hex == '00110023ef0032340000' - a.replace('0x00', '', bytealigned=True) - assert a.hex == '1123ef3234' - a.replace('0x11', '', start=1, bytealigned=True) - assert a.hex == '1123ef3234' - a.replace('0x11', '0xfff', end=7, bytealigned=True) - assert a.hex == '1123ef3234' - a.replace('0x11', '0xfff', end=8, bytealigned=True) - assert a.hex == 'fff23ef3234' - - def test_replace5(self): - a = BitStream.fromstring('0xab') - b = BitStream.fromstring('0xcd') - c = BitStream.fromstring('0xabef') - c.replace(a, b) - assert c == '0xcdef' - assert a == '0xab' - assert b == '0xcd' - a = BitStream('0x0011223344') - a.pos = 12 - a.replace('0x11', '0xfff', bytealigned=True) - assert a.pos == 0 - assert a == '0x00fff223344' - - def test_replace_with_self(self): - a = BitStream('0b11') - a.replace('0b1', a) - assert a == '0xf' - a.replace(a, a) - assert a == '0xf' - - def test_replace_count(self): - a = BitStream('0x223344223344223344') - n = a.replace('0x2', '0x0', count=0, bytealigned=True) - assert n == 0 - assert a.hex == '223344223344223344' - n = a.replace('0x2', '0x0', count=1, bytealigned=True) - assert n == 1 - assert a.hex == '023344223344223344' - n = a.replace('0x33', '', count=2, bytealigned=True) - assert n == 2 - assert a.hex == '02442244223344' - n = a.replace('0x44', '0x4444', count=1435, bytealigned=True) - assert n == 3 - assert a.hex == '02444422444422334444' - - def test_replace_bitpos(self): - a = BitStream('0xff') - a.bitpos = 8 - a.replace('0xff', '', bytealigned=True) - assert a.bitpos == 0 - a = BitStream('0b0011110001') - a.bitpos = 4 - a.replace('0b1', '0b000') - assert a.bitpos == 0 - a = BitStream('0b1') - a.bitpos = 1 - a.replace('0b1', '0b11111', bytealigned=True) - assert a.bitpos == 0 - a.replace('0b11', '0b0', False) - assert a.bitpos == 0 - a.append('0b00') - a.pos = 5 - a.replace('0b00', '0b11') - assert a.bitpos == 5 - - def test_replace_errors(self): - a = BitStream('0o123415') - with pytest.raises(ValueError): - a.replace('', Bits(0o7), bytealigned=True) - with pytest.raises(ValueError): - a.replace('0b1', '0b1', start=-100, bytealigned=True) - with pytest.raises(ValueError): - a.replace('0b1', '0b1', end=19, bytealigned=True) - - -class TestSliceAssignment: - def test_set_slice(self): - a = BitStream() - a[0:0] = '0xabcdef' - assert a.bytepos == 0 - a[4:16] = '' - assert a == '0xaef' - assert a.bitpos == 0 - a.pos = 4 - a[8:] = '0x00' - assert a == '0xae00' - assert a.bitpos == 0 - a += '0xf' - assert a.bitpos == 20 - a[8:] = '0xe' - assert a == '0xaee' - assert a.bitpos == 0 - b = BitStream("0x0000") - b[0:16] = '0xffee' - assert b == '0xffee' - b[4:16] = '0xeed123' - assert b == '0xfeed123' - b[0:8] = '0x0000' - assert b == '0x0000ed123' - a = BitStream('0xabcde') - assert a[-100:-90] == '' - assert a[-100:-16] == '0xa' - a[-100:-16] = '0x0' - assert a == '0x0bcde' - - def test_inserting_using_set_item(self): - a = BitStream() - a[0:0] = '0xdeadbeef' - assert a == '0xdeadbeef' - assert a.bytepos == 0 - a[16:16] = '0xfeed' - assert a == '0xdeadfeedbeef' - assert a.bytepos == 0 - a[0:0] = '0xa' - assert a == '0xadeadfeedbeef' - assert a.bitpos == 0 - a.bytepos = 6 - a[0:8] = '0xff' - assert a.bytepos == 6 - a[8:0] = '0x000' - assert a.startswith('0xff000ead') - - def test_slice_assignment_bit_pos(self): - a = BitStream('int:64=-1') - a.pos = 64 - a[0:8] = '' - assert a.pos == 0 - a.pos = 52 - a[-16:] = '0x0000' - assert a.pos == 52 - - - -class TestPack: - def test_pack1(self): - s = bitstring.pack('uint:6, bin, hex, int:6, se, ue, oct', 10, '0b110', 'ff', -1, -6, 6, '54') - t = BitStream('uint:6=10, 0b110, 0xff, int:6=-1, se=-6, ue=6, oct=54') - assert s == t - with pytest.raises(ValueError): - pack('tomato', '0') - with pytest.raises(ValueError): - pack('uint', 12) - with pytest.raises(ValueError): - pack('int', 12) - with pytest.raises(ValueError): - pack('hex', 'penguin') - with pytest.raises(ValueError): - pack('hex12', '0x12') - - def test_pack_with_literals(self): - s = bitstring.pack('0xf') - assert s == '0xf' - assert type(s), BitStream - s = pack('0b1') - assert s == '0b1' - s = pack('0o7') - assert s == '0o7' - s = pack('int:10=-1') - assert s == '0b1111111111' - s = pack('uint:10=1') - assert s == '0b0000000001' - s = pack('ue=12') - assert s.ue == 12 - s = pack('se=-12') - assert s.se == -12 - s = pack('bin=01') - assert s.bin == '01' - s = pack('hex=01') - assert s.hex == '01' - s = pack('oct=01') - assert s.oct == '01' - - def test_pack_with_dict(self): - a = pack('uint:6=width, se=height', height=100, width=12) - w, h = a.unpack('uint:6, se') - assert w == 12 - assert h == 100 - d = {'w': '0xf', '300': 423, 'e': '0b1101'} - a = pack('int:100=300, bin=e, uint:12=300', **d) - x, y, z = a.unpack('int:100, bin, uint:12') - assert x == 423 - assert y == '1101' - assert z == 423 - - def test_pack_with_dict2(self): - a = pack('int:5, bin:3=b, 0x3, bin=c, se=12', 10, b='0b111', c='0b1') - b = BitStream('int:5=10, 0b111, 0x3, 0b1, se=12') - assert a == b - a = pack('bits:3=b', b=BitStream('0b101')) - assert a == '0b101' - a = pack('bits:24=b', b=BitStream('0x001122')) - assert a == '0x001122' - - def test_pack_with_dict3(self): - s = pack('hex:4=e, hex:4=0xe, hex:4=e', e='f') - assert s == '0xfef' - s = pack('sep', sep='0b00') - assert s == '0b00' - - def test_pack_with_dict4(self): - s = pack('hello', hello='0xf') - assert s == '0xf' - s = pack('x, y, x, y, x', x='0b10', y='uint:12=100') - t = BitStream('0b10, uint:12=100, 0b10, uint:12=100, 0b10') - assert s == t - a = [1, 2, 3, 4, 5] - s = pack('int:8, div,' * 5, *a, **{'div': '0b1'}) - t = BitStream('int:8=1, 0b1, int:8=2, 0b1, int:8=3, 0b1, int:8=4, 0b1, int:8=5, 0b1') - assert s == t - - def test_pack_with_locals(self): - width = 352 - height = 288 - s = pack('uint:12=width, uint:12=height', **locals()) - assert s == '0x160120' - - def test_pack_with_length_restriction(self): - _ = pack('bin:3', '0b000') - with pytest.raises(bitstring.CreationError): - _ = pack('bin:3', '0b0011') - with pytest.raises(bitstring.CreationError): - _ = pack('bin:3', '0b11') - with pytest.raises(bitstring.CreationError): - _ = pack('bin:3=0b0011') - with pytest.raises(bitstring.CreationError): - _ = pack('bin:3=0b11') - - _ = pack('hex:4', '0xf') - with pytest.raises(bitstring.CreationError): - _ = pack('hex:4', '0b111') - with pytest.raises(bitstring.CreationError): - _ = pack('hex:4', '0b11111') - with pytest.raises(bitstring.CreationError): - _ = pack('hex:8=0xf') - - _ = pack('oct:6', '0o77') - with pytest.raises(bitstring.CreationError): - _ = pack('oct:6', '0o1') - with pytest.raises(bitstring.CreationError): - _ = pack('oct:6', '0o111') - with pytest.raises(bitstring.CreationError): - _ = pack('oct:3', '0b1') - with pytest.raises(bitstring.CreationError): - _ = pack('oct:3=hello', hello='0o12') - - _ = pack('bits:3', BitStream('0b111')) - with pytest.raises(bitstring.CreationError): - _ = pack('bits:3', BitStream('0b11')) - with pytest.raises(bitstring.CreationError): - _ = pack('bits:3', BitStream('0b1111')) - with pytest.raises(bitstring.CreationError): - _ = pack('bits:12=b', b=BitStream('0b11')) - - def test_pack_null(self): - s = pack('') - assert not s - s = pack(',') - assert not s - s = pack(',,,,,0b1,,,,,,,,,,,,,0b1,,,,,,,,,,') - assert s == '0b11' - s = pack(',,uint:12,,bin:3,', 100, '100') - a, b = s.unpack('uint:12,bin:3') - assert a == 100 - assert b == '100' - - def test_pack_uint(self): - s = pack('uint:10, uint:5', 1, 2) - a, b = s.unpack('10, 5') - assert (a.uint, b.uint) == (1, 2) - s = pack('uint:10=150, uint:12=qee', qee=3) - assert s == 'uint:10=150, uint:12=3' - t = BitStream('uint:100=5') - assert t == 'uint:100=5' - - def test_pack_defualt_uint_errors(self): - with pytest.raises(bitstring.CreationError): - _ = BitStream('5=-1') - - def test_packing_long_keyword_bitstring(self): - s = pack('bits=b', b=BitStream(128000)) - assert s == BitStream(128000) - - def test_packing_with_list_format(self): - f = ['bin', 'hex', 'uint:10'] - a = pack(','.join(f), '00', '234', 100) - b = pack(f, '00', '234', 100) - assert a == b - - -class TestUnpack: - def test_unpack1(self): - s = BitStream('uint:13=23, hex=e, bin=010, int:41=-554, 0o44332, se=-12, ue=4') - s.pos = 11 - a, b, c, d, e, f, g = s.unpack('uint:13, hex:4, bin:3, int:41, oct:15, se, ue') - assert a == 23 - assert b == 'e' - assert c == '010' - assert d == -554 - assert e == '44332' - assert f == -12 - assert g == 4 - assert s.pos == 11 - - def test_unpack2(self): - s = BitStream('0xff, 0b000, uint:12=100') - a, b, c = s.unpack('bits:8, bits, uint:12') - assert type(s) == BitStream - assert a == '0xff' - assert type(s) == BitStream - assert b == '0b000' - assert c == 100 - a, b = s.unpack(['bits:11', 'uint']) - assert a == '0xff, 0b000' - assert b == 100 - - -class TestFromFile: - def test_creation_from_file_operations(self): - filename = os.path.join(THIS_DIR, 'smalltestfile') - s = BitStream(filename=filename) - s.append('0xff') - assert s.hex == '0123456789abcdefff' - - s = ConstBitStream(filename=filename) - t = BitStream('0xff') + s - assert t.hex == 'ff0123456789abcdef' - - s = BitStream(filename=filename) - del s[:1] - assert (BitStream('0b0') + s).hex == '0123456789abcdef' - - s = BitStream(filename=filename) - del s[:7 * 8] - assert s.hex == 'ef' - - s = BitStream(filename=filename) - s.insert('0xc', 4) - assert s.hex == '0c123456789abcdef' - - s = BitStream(filename=filename) - s.prepend('0xf') - assert s.hex == 'f0123456789abcdef' - - s = BitStream(filename=filename) - s.overwrite('0xaaa', 12) - assert s.hex == '012aaa6789abcdef' - - s = BitStream(filename=filename) - s.reverse() - assert s.hex == 'f7b3d591e6a2c480' - - s = BitStream(filename=filename) - del s[-60:] - assert s.hex == '0' - - s = BitStream(filename=filename) - del s[:60] - assert s.hex == 'f' - - def test_file_properties(self): - s = ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile')) - assert s.hex == '0123456789abcdef' - assert s.uint == 81985529216486895 - assert s.int == 81985529216486895 - assert s.bin == '0000000100100011010001010110011110001001101010111100110111101111' - assert s[:-1].oct == '002215053170465363367' - s.bitpos = 0 - assert s.read('se') == -72 - s.bitpos = 0 - assert s.read('ue') == 144 - assert s.bytes == b'\x01\x23\x45\x67\x89\xab\xcd\xef' - assert s.tobytes() == b'\x01\x23\x45\x67\x89\xab\xcd\xef' - - def test_creation_from_file_with_length(self): - test_filename = os.path.join(THIS_DIR, 'test.m1v') - s = ConstBitStream(filename=test_filename, length=32) - assert s.length == 32 - assert s.hex == '000001b3' - assert s.bytes == b'\x00\x00\x01\xb3' - assert s.uint == 0x1b3 - assert s.int == 0x1b3 - assert s.bin == '00000000000000000000000110110011' - s = ConstBitStream(filename=test_filename, length=0) - assert not s - small_test_filename = os.path.join(THIS_DIR, 'smalltestfile') - with pytest.raises(bitstring.CreationError): - _ = BitStream(filename=small_test_filename, length=65) - with pytest.raises(bitstring.CreationError): - _ = ConstBitStream(filename=small_test_filename, length=64, offset=1) - with pytest.raises(bitstring.CreationError): - _ = ConstBitStream(filename=small_test_filename, offset=65) - with open(small_test_filename, 'rb') as f: - with pytest.raises(bitstring.CreationError): - _ = ConstBitStream(f, offset=65) - with pytest.raises(bitstring.CreationError): - _ = ConstBitStream(f, length=65) - with pytest.raises(bitstring.CreationError): - _ = ConstBitStream(f, offset=60, length=5) - - def test_creation_from_file_with_offset(self): - filename = os.path.join(THIS_DIR, 'test.m1v') - a = BitStream(filename=filename, offset=4) - assert a.peek(4 * 8).hex == '00001b31' - b = BitStream(filename=filename, offset=28) - assert b.peek(8).hex == '31' - - def test_file_slices(self): - s = BitStream(filename=os.path.join(THIS_DIR, 'smalltestfile')) - assert s[-16:].hex == 'cdef' - - def test_creataion_from_file_errors(self): - with pytest.raises(IOError): - _ = BitStream(filename='Idonotexist') - - def test_find_in_file(self): - s = BitStream(filename=os.path.join(THIS_DIR, 'test.m1v')) - assert s.find('0x160120') - assert s.bytepos == 4 - s3 = s.read(24) - assert s3.hex == '160120' - s.bytepos = 0 - assert s._pos == 0 - assert s.find('0x0001b2') - assert s.bytepos == 13 - - def test_hex_from_file(self): - s = BitStream(filename=os.path.join(THIS_DIR, 'test.m1v')) - assert s[0:32].hex == '000001b3' - assert s[-32:].hex == '000001b7' - s.hex = '0x11' - assert s.hex == '11' - - def test_file_operations(self): - filename = os.path.join(THIS_DIR, 'test.m1v') - s1 = BitStream(filename=filename) - s2 = BitStream(filename=filename) - assert s1.read(32).hex == '000001b3' - assert s2.read(32).hex == '000001b3' - s1.bytepos += 4 - assert s1.read(8).hex == '02' - assert s2.read(5 * 8).hex == '1601208302' - s1.pos = s1.len - with pytest.raises(ValueError): - s1.pos += 1 - - def test_file_bit_getting(self): - s = ConstBitStream(filename=os.path.join(THIS_DIR, 'smalltestfile'), offset=16, length=8) - b = s[1] - assert b - b = s.any(0, [-1, -2, -3]) - assert b - b = s.all(0, [0, 1, 2]) - assert not b - - -class TestCreationErrors: - def test_incorrect_bin_assignment(self): - s = BitStream() - with pytest.raises(bitstring.CreationError): - s._setbin('0010020') - - def test_incorrect_hex_assignment(self): - s = BitStream() - with pytest.raises(bitstring.CreationError): - s.hex = '0xabcdefg' - - -class TestLength: - def test_length_zero(self): - assert BitStream('').len == 0 - - def test_length(self): - assert BitStream('0x80').len == 8 - - def test_offset_length_error(self): - with pytest.raises(bitstring.CreationError): - BitStream(hex='0xffff', offset=-1) - - -class TestSimpleConversions: - def test_convert_to_uint(self): - assert BitStream('0x10').uint == 16 - assert BitStream('0b000111').uint == 7 - - def test_convert_to_int(self): - assert BitStream('0x10').int == 16 - assert BitStream('0b11110').int == -2 - - def test_convert_to_hex(self): - assert BitStream(bytes=b'\x00\x12\x23\xff').hex == '001223ff' - s = BitStream('0b11111') - with pytest.raises(bitstring.InterpretError): - _ = s.hex - - -class TestEmpty: - def test_empty_bitstring(self): - s = BitStream() - with pytest.raises(bitstring.ReadError): - s.read(1) - assert s.bin == '' - assert s.hex == '' - with pytest.raises(bitstring.InterpretError): - _ = s.int - with pytest.raises(bitstring.InterpretError): - _ = s.uint - assert not s - - def test_non_empty_bit_stream(self): - s = BitStream(bin='0') - assert not not s.len - - -class TestPosition: - def test_bit_position(self): - s = BitStream(bytes=b'\x00\x00\x00') - assert s.bitpos == 0 - s.read(5) - assert s.pos == 5 - s.pos = s.len - with pytest.raises(bitstring.ReadError): - s.read(1) - - def test_byte_position(self): - s = BitStream(bytes=b'\x00\x00\x00') - assert s.bytepos == 0 - s.read(10) - with pytest.raises(bitstring.ByteAlignError): - _ = s.bytepos - s.read(6) - assert s.bytepos == 2 - - def test_seek_to_bit(self): - s = BitStream(bytes=b'\x00\x00\x00\x00\x00\x00') - s.bitpos = 0 - assert s.bitpos == 0 - with pytest.raises(ValueError): - s.pos = -1 - with pytest.raises(ValueError): - s.bitpos = 6 * 8 + 1 - s.bitpos = 6 * 8 - assert s.bitpos == 6 * 8 - - def test_seek_to_byte(self): - s = BitStream(bytes=b'\x00\x00\x00\x00\x00\xab') - s.bytepos = 5 - assert s.read(8).hex == 'ab' - - def test_advance_bits_and_bytes(self): - s = BitStream(bytes=b'\x00\x00\x00\x00\x00\x00\x00\x00') - s.pos += 5 - assert s.pos == 5 - s.bitpos += 16 - assert s.pos == 2 * 8 + 5 - s.pos -= 8 - assert s.pos == 8 + 5 - - def test_retreat_bits_and_bytes(self): - a = BitStream(length=100) - a.pos = 80 - a.bytepos -= 5 - assert a.bytepos == 5 - a.pos -= 5 - assert a.pos == 35 - - -class TestAppend: - def test_append(self): - s1 = BitStream('0b00000') - s1.append(BitStream(bool=True)) - assert s1.bin == '000001' - assert (BitStream('0x0102') + BitStream('0x0304')).hex == '01020304' - - def test_append_same_bitstring(self): - s1 = BitStream('0xf0')[:6] - s1.append(s1) - assert s1.bin == '111100111100' - - def test_append_with_offset(self): - s = BitStream(bytes=b'\x28\x28', offset=1) - s.append('0b0') - assert s.hex == '5050' - - -class TestByteAlign: - def test_byte_align(self): - s = BitStream(hex='0001ff23') - s.bytealign() - assert s.bytepos == 0 - s.pos += 11 - s.bytealign() - assert s.bytepos == 2 - s.pos -= 10 - s.bytealign() - assert s.bytepos == 1 - - def test_insert_byte_aligned(self): - s = BitStream('0x0011') - s.insert(BitStream('0x22'), 8) - assert s.hex == '002211' - s = BitStream(0) - s.insert(BitStream(bin='101'), 0) - assert s.bin == '101' - - -class TestTruncate: - def test_truncate_start(self): - s = BitStream('0b1') - del s[0] - assert not s - s = BitStream(hex='1234') - assert s.hex == '1234' - del s[:4] - assert s.hex == '234' - del s[:9] - assert s.bin == '100' - del s[:2] - assert s.bin == '0' - assert s.len == 1 - del s[:1] - assert not s - - def test_truncate_end(self): - s = BitStream('0b1') - del s[-1:] - assert not s - s = BitStream(bytes=b'\x12\x34') - assert s.hex == '1234' - del s[-4:] - assert s.hex == '123' - del s[-9:] - assert s.bin == '000' - del s[-3:] - assert not s - s = BitStream('0b001') - del s[:2] - del s[-1:] - assert not s - - -class TestSlice: - def test_byte_aligned_slice(self): - s = BitStream(hex='0x123456') - assert s[8:16].hex == '34' - s = s[8:24] - assert s.len == 16 - assert s.hex == '3456' - s = s[0:8] - assert s.hex == '34' - s.hex = '0x123456' - assert s[8:24][0:8].hex == '34' - - def test_slice(self): - s = BitStream(bin='000001111100000') - s1 = s[0:5] - s2 = s[5:10] - s3 = s[10:15] - assert s1.bin == '00000' - assert s2.bin == '11111' - assert s3.bin == '00000' - - -class TestInsert: - def test_insert(self): - s1 = BitStream(hex='0x123456') - s2 = BitStream(hex='0xff') - s1.bytepos = 1 - s1.insert(s2) - assert s1.bytepos == 2 - assert s1.hex == '12ff3456' - s1.insert('0xee', 24) - assert s1.hex == '12ff34ee56' - assert s1.bitpos == 32 - with pytest.raises(ValueError): - s1.insert('0b1', -1000) - with pytest.raises(ValueError): - s1.insert('0b1', 1000) - - def test_insert_null(self): - s = BitStream(hex='0x123') - s.insert(BitStream(), 3) - assert s.hex == '123' - - def test_insert_bits(self): - one = BitStream(bin='1') - zero = BitStream(bin='0') - s = BitStream(bin='00') - s.insert(one, 0) - assert s.bin == '100' - s.insert(zero, 0) - assert s.bin == '0100' - s.insert(one, s.len) - assert s.bin == '01001' - s.insert(s, 2) - assert s.bin == '0101001001' - - -class TestResetting: - def test_set_hex(self): - s = BitStream() - s.hex = '0' - assert s.hex == '0' - s.hex = '0x010203045' - assert s.hex == '010203045' - with pytest.raises(bitstring.CreationError): - s.hex = '0x002g' - - def test_set_bin(self): - s = BitStream(bin="000101101") - assert s.bin == '000101101' - assert s.len == 9 - s.bin = '0' - assert s.bin == '0' - assert s.len == 1 - - def test_set_empty_bin(self): - s = BitStream(hex='0x000001b3') - s.bin = '' - assert s.len == 0 - assert s.bin == '' - - def test_set_invalid_bin(self): - s = BitStream() - with pytest.raises(bitstring.CreationError): - s.bin = '00102' - - -class TestOverwriting: - def test_overwrite_bit(self): - s = BitStream(bin='0') - s.overwrite(BitStream(bin='1'), 0) - assert s.bin == '1' - - def test_overwrite_limits(self): - s = BitStream(bin='0b11111') - s.overwrite(BitStream(bin='000'), 0) - assert s.bin == '00011' - s.overwrite('0b000', 2) - assert s.bin == '00000' - - def test_overwrite_null(self): - s = BitStream(hex='342563fedec') - s2 = BitStream(s) - s.overwrite(BitStream(bin=''), 23) - assert s.bin == s2.bin - - def test_overwrite_position(self): - s1 = BitStream(hex='0123456') - s2 = BitStream(hex='ff') - s1.bytepos = 1 - s1.overwrite(s2) - assert (s1.hex, s1.bytepos) == ('01ff456', 2) - s1.overwrite('0xff', 0) - assert (s1.hex, s1.bytepos) == ('ffff456', 1) - - def test_overwrite_with_self(self): - s = BitStream('0x123') - s.overwrite(s) - assert s == '0x123' - - -class TestSplit: - def test_split_byte_aligned_corner_cases(self): - s = BitStream() - bsl = s.split(BitStream(hex='0xff')) - assert next(bsl).hex == '' - with pytest.raises(StopIteration): - _ = next(bsl) - s = BitStream(hex='aabbcceeddff') - delimiter = BitStream() - bsl = s.split(delimiter) - with pytest.raises(ValueError): - _ = next(bsl) - delimiter = BitStream(hex='11') - bsl = s.split(delimiter) - assert next(bsl).hex == s.hex - - def test_split_byte_aligned(self): - s = BitStream(hex='0x1234aa1234bbcc1234ffff') - delimiter = BitStream(hex='1234') - bsl = s.split(delimiter) - assert [b.hex for b in bsl] == ['', '1234aa', '1234bbcc', '1234ffff'] - assert s.pos == 0 - - def test_split_byte_aligned_with_intial_bytes(self): - s = BitStream(hex='aa471234fedc43 47112233 47 4723 472314') - delimiter = BitStream(hex='47') - s.find(delimiter) - assert s.bytepos == 1 - bsl = s.split(delimiter, start=0) - assert [b.hex for b in bsl] == ['aa', '471234fedc43', '47112233', - '47', '4723', '472314'] - assert s.bytepos == 1 - - def test_split_byte_aligned_with_overlapping_delimiter(self): - s = BitStream(hex='aaffaaffaaffaaffaaff') - bsl = s.split(BitStream(hex='aaffaa')) - assert [b.hex for b in bsl] == ['', 'aaffaaff', 'aaffaaffaaff'] - - -class TestAdding: - def test_adding(self): - s1 = BitStream(hex='0x0102') - s2 = BitStream(hex='0x0304') - s3 = s1 + s2 - assert s1.hex == '0102' - assert s2.hex == '0304' - assert s3.hex == '01020304' - s3 += s1 - assert s3.hex == '010203040102' - assert s2[9:16].bin == '0000100' - assert s1[0:9].bin == '000000010' - s4 = BitStream(bin='000000010') + BitStream(bin='0000100') - assert s4.bin == '0000000100000100' - s5 = s1[0:9] + s2[9:16] - assert s5.bin == '0000000100000100' - - def test_more_adding(self): - s = BitStream(bin='00') + BitStream(bin='') + BitStream(bin='11') - assert s.bin == '0011' - s = '0b01' - s += BitStream('0b11') - assert s.bin == '0111' - s = BitStream('0x00') - t = BitStream('0x11') - s += t - assert s.hex == '0011' - assert t.hex == '11' - s += s - assert s.hex == '00110011' - - def test_radd(self): - s = '0xff' + BitStream('0xee') - assert s.hex == 'ffee' - - def test_truncate_asserts(self): - s = BitStream('0x001122') - s.bytepos = 2 - del s[-s.len:] - # self.assertEqual(s.bytepos, 0) - s.append('0x00') - s.append('0x1122') - s.bytepos = 2 - del s[:s.len] - # self.assertEqual(s.bytepos, 0) - s.append('0x00') - - def test_overwrite_errors(self): - s = BitStream(bin='11111') - with pytest.raises(ValueError): - s.overwrite(BitStream(bin='1'), -10) - with pytest.raises(ValueError): - s.overwrite(BitStream(bin='1'), 6) - s.overwrite('bin=0', 5) - assert s.b == '111110' - s.overwrite(BitStream(hex='0x00'), 1) - assert s.b == '100000000' - - def test_delete_bits(self): - s = BitStream(bin='000111100000') - s.bitpos = 4 - del s[4:8] - assert s.bin == '00010000' - del s[4:1004] - assert s.bin, '0001' - - def test_delete_bits_with_position(self): - s = BitStream(bin='000111100000') - del s[4:8] - assert s.bin == '00010000' - - def test_delete_bytes(self): - s = BitStream('0x00112233') - del s[8:8] - assert s.hex == '00112233' - assert s.pos == 0 - del s[8:16] - assert s.hex == '002233' - assert s.bytepos == 0 - del s[:24] - assert not s - assert s.pos == 0 - - def test_get_item_with_positive_position(self): - s = BitStream(bin='0b1011') - assert s[0] == True - assert s[1] == False - assert s[2] == True - assert s[3] == True - with pytest.raises(IndexError): - _ = s[4] - - def test_get_item_with_negative_position(self): - s = BitStream(bin='1011') - assert s[-1] == True - assert s[-2] == True - assert s[-3] == False - assert s[-4] == True - with pytest.raises(IndexError): - _ = s[-5] - - def test_slicing(self): - s = ConstBitStream(hex='0123456789') - assert s[0:8].hex == '01' - assert not s[0:0] - assert not s[23:20] - assert s[8:12].bin == '0010' - assert s[32:80] == '0x89' - - def test_negative_slicing(self): - s = ConstBitStream(hex='012345678') - assert s[:-8].hex == '0123456' - assert s[-16:-8].hex == '56' - assert s[-24:].hex == '345678' - assert s[-1000:-24] == '0x012' - - def test_len(self): - s = BitStream() - assert len(s) == 0 - s.append(BitStream(bin='001')) - assert len(s) == 3 - - def test_join(self): - s1 = BitStream(bin='0') - s2 = BitStream(bin='1') - s3 = BitStream(bin='000') - s4 = BitStream(bin='111') - strings = [s1, s2, s1, s3, s4] - s = BitStream().join(strings) - assert s.bin == '010000111' - - def test_join2(self): - s1 = BitStream(hex='00112233445566778899aabbccddeeff') - s2 = BitStream(bin='0b000011') - bsl = [s1[0:32], s1[4:12], s2, s2, s2, s2] - s = ConstBitStream().join(bsl) - assert s.hex == '00112233010c30c3' - - bsl = [BitStream(uint=j, length=12) for j in range(10) for _ in range(10)] - s = BitStream().join(bsl) - assert s.length == 1200 - - def test_join_with_ints(self): - with pytest.raises(TypeError): - s = BitStream().join([1, 2]) - - def test_pos(self): - s = BitStream(bin='1') - assert s.bitpos == 0 - s.read(1) - assert s.bitpos == 1 - - def test_writing_data(self): - strings = [BitStream(bin=x) for x in ['0', '001', '0011010010', '010010', '1011']] - s = BitStream().join(strings) - s2 = BitStream(bytes=s.bytes) - assert s2.bin == '000100110100100100101011' - s2.append(BitStream(bin='1')) - s3 = BitStream(bytes=s2.tobytes()) - assert s3.bin == '00010011010010010010101110000000' - - def test_writing_data_with_offsets(self): - s1 = BitStream(bytes=b'\x10') - s2 = BitStream(bytes=b'\x08\x00', length=8, offset=1) - s3 = BitStream(bytes=b'\x04\x00', length=8, offset=2) - assert s1 == s2 - assert s2 == s3 - assert s1.bytes == s2.bytes - assert s2.bytes == s3.bytes - - def test_various_things1(self): - hexes = ['12345678', '87654321', 'ffffffffff', 'ed', '12ec'] - bins = ['001010', '1101011', '0010000100101110110110', '11', '011'] - bsl = [] - for (hex_, bin_) in list(zip(hexes, bins)) * 5: - bsl.append(BitStream(hex=hex_)) - bsl.append(BitStream(bin=bin_)) - s = BitStream().join(bsl) - for (hex_, bin_) in list(zip(hexes, bins)) * 5: - h = s.read(4 * len(hex_)) - b = s.read(len(bin_)) - assert h.hex == hex_ - assert b.bin == bin_ - - def test_various_things2(self): - s1 = BitStream(hex="0x1f08")[:13] - assert s1.bin == '0001111100001' - s2 = BitStream(bin='0101') - assert s2.bin == '0101' - s1.append(s2) - assert s1.length == 17 - assert s1.bin == '00011111000010101' - s1 = s1[3:8] - assert s1.bin == '11111' - - def test_various_things3(self): - s1 = BitStream(hex='0x012480ff')[2:27] - s2 = s1 + s1 - assert s2.length == 50 - s3 = s2[0:25] - s4 = s2[25:50] - assert s3.bin == s4.bin - - def test_peek_bit(self): - s = BitStream(bin='01') - assert s.peek(1) == [0] - assert s.peek(1) == [0] - assert s.read(1) == [0] - assert s.peek(1) == [1] - assert s.peek(1) == [1] - - s = BitStream(bytes=b'\x1f', offset=3) - assert s.len == 5 - assert s.peek(5).bin == '11111' - assert s.peek(5).bin == '11111' - s.pos += 1 - with pytest.raises(bitstring.ReadError): - _ = s.peek(5) - - s = BitStream(hex='001122334455') - assert s.peek(8).hex == '00' - assert s.read(8).hex == '00' - s.pos += 33 - with pytest.raises(bitstring.ReadError): - _ = s.peek(8) - - s = BitStream(hex='001122334455') - assert s.peek(8 * 2).hex == '0011' - assert s.read(8 * 3).hex == '001122' - assert s.peek(8 * 3).hex == '334455' - with pytest.raises(bitstring.ReadError): - _ = s.peek(25) - - def test_advance_bit(self): - s = BitStream(hex='0xff') - s.bitpos = 6 - s.pos += 1 - assert s.bitpos == 7 - s.bitpos += 1 - with pytest.raises(ValueError): - s.pos += 1 - - def test_advance_byte(self): - s = BitStream(hex='0x010203') - s.bytepos += 1 - assert s.bytepos == 1 - s.bytepos += 1 - assert s.bytepos == 2 - s.bytepos += 1 - with pytest.raises(ValueError): - s.bytepos += 1 - - def test_retreat_bit(self): - s = BitStream(hex='0xff') - with pytest.raises(ValueError): - s.pos -= 1 - s.pos = 5 - s.pos -= 1 - assert s.pos == 4 - - def test_retreat_byte(self): - s = BitStream(hex='0x010203') - with pytest.raises(ValueError): - s.bytepos -= 1 - s.bytepos = 3 - s.bytepos -= 1 - assert s.bytepos == 2 - assert s.read(8).hex == '03' - - def test_creation_by_auto(self): - s = BitStream('0xff') - assert s.hex == 'ff' - s = BitStream('0b00011') - assert s.bin == '00011' - with pytest.raises(bitstring.CreationError): - _ = BitStream('hello') - s1 = BitStream(bytes=b'\xf5', length=3, offset=5) - with pytest.raises(TypeError): - _ = BitStream(1.2) - - def test_creation_by_auto2(self): - s = BitStream('bin=001') - assert s.bin == '001' - s = BitStream('oct=0o007') - assert s.oct == '007' - s = BitStream('hex=123abc') - assert s == '0x123abc' - - s = BitStream('bin2=01') - assert s == '0b01' - for s in ['bin:1=01', 'bits:4=0b1', 'oct3=000', 'hex4=0x1234']: - with pytest.raises(bitstring.CreationError): - _ = BitStream(s) - - def test_insert_using_auto(self): - s = BitStream('0xff') - s.insert('0x00', 4) - assert s.hex == 'f00f' - with pytest.raises(ValueError): - s.insert('ff') - - def test_overwrite_using_auto(self): - s = BitStream('0x0110') - s.overwrite('0b1') - assert s.hex == '8110' - s.overwrite('') - assert s.hex == '8110' - with pytest.raises(ValueError): - s.overwrite('0bf') - - def test_find_using_auto(self): - s = BitStream('0b000000010100011000') - assert s.find('0b101') - assert s.pos == 7 - - def test_findbytealigned_using_auto(self): - s = BitStream('0x00004700') - assert s.find('0b01000111', bytealigned=True) - assert s.bytepos == 2 - - def test_append_using_auto(self): - s = BitStream('0b000') - s.append('0b111') - assert s.bin == '000111' - s.append('0b0') - assert s.bin == '0001110' - - def test_split_byte_aligned_using_auto(self): - s = BitStream('0x000143563200015533000123') - sections = s.split('0x0001') - assert next(sections).hex == '' - assert next(sections).hex == '0001435632' - assert next(sections).hex == '00015533' - assert next(sections).hex == '000123' - pytest.raises(StopIteration, next, sections) - - def test_split_byte_aligned_with_self(self): - s = BitStream('0x1234') - sections = s.split(s) - assert next(sections).hex == '' - assert next(sections).hex == '1234' - with pytest.raises(StopIteration): - next(sections) - - def test_prepend(self): - s = BitStream('0b000') - s.prepend('0b11') - assert s.bin == '11000' - s.prepend(s) - assert s.bin == '1100011000' - s.prepend('') - assert s.bin == '1100011000' - - def test_null_slice(self): - s = BitStream('0x111') - t = s[1:1] - assert len(t) == 0 - - def test_multiple_autos(self): - s = BitStream('0xa') - s.prepend('0xf') - s.append('0xb') - assert s == '0xfab' - s.prepend(s) - s.append('0x100') - s.overwrite('0x5', 4) - assert s == '0xf5bfab100' - - def test_reverse(self): - s = BitStream('0b0011') - s.reverse() - assert s.bin == '1100' - s = BitStream('0b10') - s.reverse() - assert s.bin == '01' - s = BitStream() - s.reverse() - assert s.bin == '' - - def test_init_with_concatenated_strings(self): - s = BitStream('0xff 0Xee 0xd 0xcc') - assert s.hex == 'ffeedcc' - s = BitStream('0b0 0B111 0b001') - assert s.bin == '0111001' - s += '0b1' + '0B1' - assert s.bin == '011100111' - s = BitStream(hex='ff0xee') - assert s.hex == 'ffee' - s = BitStream(bin='000b0b11') - assert s.bin == '0011' - s = BitStream(' 0o123 0O 7 0 o1') - assert s.oct == '12371' - s += ' 0 o 332' - assert s.oct == '12371332' - - def test_equals(self): - s1 = BitStream('0b01010101') - s2 = BitStream('0b01010101') - assert s1 == s2 - s3 = BitStream() - s4 = BitStream() - assert s3 == s4 - assert not s3 != s4 - s5 = BitStream(bytes=b'\xff', offset=2, length=3) - s6 = BitStream('0b111') - assert s5 == s6 - - class A(object): - pass - assert not s5 == A() - - def test_large_equals(self): - s1 = BitStream(1000000) - s2 = BitStream(1000000) - s1.set(True, [-1, 55, 53214, 534211, 999999]) - s2.set(True, [-1, 55, 53214, 534211, 999999]) - assert s1 == s2 - s1.set(True, 800000) - assert s1 != s2 - - def test_not_equals(self): - s1 = BitStream('0b0') - s2 = BitStream('0b1') - assert s1 != s2 - assert not s1 != BitStream('0b0') - - def test_equality_with_auto_initialised(self): - a = BitStream('0b00110111') - assert a == '0b00110111' - assert a == '0x37' - assert '0b0011 0111' == a - assert '0x3 0x7' == a - assert not a == '0b11001000' - assert not '0x3737' == a - - def test_invert_special_method(self): - s = BitStream('0b00011001') - t = ~s - assert t.bin == '11100110' - assert (~BitStream('0b0')).bin == '1' - assert (~BitStream('0b1')).bin == '0' - u = ~t - assert u == s - - def test_invert_bit_position(self): - s = ConstBitStream('0xefef') - s.pos = 8 - t = ~s - assert s.pos == 8 - assert t.pos == 0 - - def test_invert_special_method_errors(self): - s = BitStream() - with pytest.raises(bitstring.Error): - _ = ~s - - def test_join_with_auto(self): - s = BitStream().join(['0xf', '0b00', BitStream(bin='11')]) - assert s == '0b11110011' - - def test_auto_bit_string_copy(self): - s = BitStream('0xabcdef') - t = BitStream(s) - assert t.hex == 'abcdef' - del s[-8:] - assert t.hex == 'abcdef' - - -class TestMultiplication: - - def test_multiplication(self): - a = BitStream('0xff') - b = a * 8 - assert b == '0xffffffffffffffff' - b = 4 * a - assert b == '0xffffffff' - assert 1 * a == a * 1 == a - c = a * 0 - assert not c - a *= 3 - assert a == '0xffffff' - a *= 0 - assert not a - one = BitStream('0b1') - zero = BitStream('0b0') - mix = one * 2 + 3 * zero + 2 * one * 2 - assert mix == '0b110001111' - q = BitStream() - q *= 143 - assert not q - q += [True, True, False] - assert q.bitpos == 3 - q *= 0 - assert not q - - def test_multiplication_with_files(self): - a = BitStream(filename=os.path.join(THIS_DIR, 'test.m1v')) - b = a.len - a *= 3 - assert a.len == 3 * b - - def test_multiplication_errors(self): - a = BitStream('0b1') - b = BitStream('0b0') - with pytest.raises(ValueError): - _ = a * -1 - with pytest.raises(ValueError): - a *= -1 - with pytest.raises(ValueError): - _ = -1 * a - with pytest.raises(TypeError): - _ = a * 1.2 - with pytest.raises(TypeError): - _ = b * a - with pytest.raises(TypeError): - a *= b - - -class TestBitWise: - - def test_bitwise_and(self): - a = BitStream('0b01101') - b = BitStream('0b00110') - assert (a & b).bin == '00100' - assert (a & '0b11111') == a - with pytest.raises(ValueError): - _ = a & '0b1' - with pytest.raises(ValueError): - _ = b & '0b110111111' - c = BitStream('0b0011011') - c.pos = 4 - d = c & '0b1111000' - assert d.pos == 0 - assert d.bin == '0011000' - d = '0b1111000' & c - assert d.bin == '0011000' - - def test_bitwise_or(self): - a = BitStream('0b111001001') - b = BitStream('0b011100011') - c = a | b - assert c.bin == '111101011' - assert (a | '0b000000000') == a - with pytest.raises(ValueError): - _ = a | '0b0000' - with pytest.raises(ValueError): - _ = b | (a + '0b1') - a = '0xff00' | BitStream('0x00f0') - assert a.hex == 'fff0' - - def test_bitwise_xor(self): - a = BitStream('0b111001001') - b = BitStream('0b011100011') - c = a ^ b - assert c.bin == '100101010' - assert (a ^ '0b111100000').bin == '000101001' - with pytest.raises(ValueError): - _ = a ^ '0b0000' - with pytest.raises(ValueError): - _ = b ^ (a + '0b1') - a = '0o707' ^ BitStream('0o777') - assert a.oct == '070' - - -class TestSplit2: - - def test_split(self): - a = BitStream('0b0 010100111 010100 0101 010') - a.pos = 20 - subs = [i.bin for i in a.split('0b010')] - assert subs == ['0', '010100111', '010100', '0101', '010'] - assert a.pos == 20 - - def test_split_corner_cases(self): - a = BitStream('0b000000') - bsl = a.split('0b1', False) - assert next(bsl) == a - with pytest.raises(StopIteration): - next(bsl) - b = BitStream() - bsl = b.split('0b001', False) - assert not next(bsl) - with pytest.raises(StopIteration): - _ = next(bsl) - - def test_split_errors(self): - a = BitStream('0b0') - b = a.split('', False) - with pytest.raises(ValueError): - _ = next(b) - - def test_slice_with_offset(self): - a = BitStream(bytes=b'\x00\xff\x00', offset=7) - b = a[7:12] - assert b.bin == '11000' - - def test_split_with_maxsplit(self): - a = BitStream('0xaabbccbbccddbbccddee') - assert len(list(a.split('0xbb', bytealigned=True))) == 4 - bsl = list(a.split('0xbb', count=1, bytealigned=True)) - assert (len(bsl), bsl[0]) == (1, '0xaa') - bsl = list(a.split('0xbb', count=2, bytealigned=True)) - assert len(bsl) == 2 - assert bsl[0] == '0xaa' - assert bsl[1] == '0xbbcc' - - def test_split_more(self): - s = BitStream('0b1100011001110110') - for i in range(10): - a = list(s.split('0b11', False, count=i)) - b = list(s.split('0b11', False))[:i] - assert a == b - b = s.split('0b11', count=-1) - with pytest.raises(ValueError): - _ = next(b) - - def test_split_startbit(self): - a = BitStream('0b0010101001000000001111') - bsl = a.split('0b001', bytealigned=False, start=1) - assert [x.bin for x in bsl] == ['010101', '001000000', '001111'] - b = a.split('0b001', start=-100) - with pytest.raises(ValueError): - _ = next(b) - b = a.split('0b001', start=23) - with pytest.raises(ValueError): - _ = next(b) - b = a.split('0b1', start=10, end=9) - with pytest.raises(ValueError): - _ = next(b) - - def test_split_startbit_byte_aligned(self): - a = BitStream('0x00ffffee') - bsl = list(a.split('0b111', start=9, bytealigned=True)) - assert [x.bin for x in bsl] == ['1111111', '11111111', '11101110'] - - def test_split_endbit(self): - a = BitStream('0b000010001001011') - bsl = list(a.split('0b1', bytealigned=False, end=14)) - assert [x.bin for x in bsl] == ['0000', '1000', '100', '10', '1'] - assert list(a[4:12].split('0b0', False)) == list(a.split('0b0', start=4, end=12)) - try: - list(a.split('0xffee', end=15)) - except ValueError: - pytest.fail("ValueError raised unexpectedly") - # Whereas this one will when we call next() - bsl = a.split('0xffee', end=16) - with pytest.raises(ValueError): - _ = next(bsl) - - def test_split_endbit_byte_aligned(self): - a = BitStream('0xff00ff')[:22] - bsl = list(a.split('0b 0000 0000 111', end=19)) - assert [x.bin for x in bsl] == ['11111111', '00000000111'] - bsl = list(a.split('0b 0000 0000 111', end=18)) - assert [x.bin for x in bsl] == ['111111110000000011'] - - def test_split_max_split(self): - a = BitStream('0b1' * 20) - for i in range(10): - bsl = list(a.split('0b1', count=i)) - assert len(bsl) == i - - ####################### - - def test_explicit_auto(self): - with pytest.raises(bitstring.CreationError): - a = BitStream(auto='0x1') - - def test_position_in_slice(self): - a = BitStream('0x00ffff00') - a.bytepos = 2 - b = a[8:24] - assert b.bytepos == 0 - - def test_find_byte_aligned_with_bits(self): - a = BitStream('0x00112233445566778899') - a.find('0b0001', bytealigned=True) - assert a.bitpos == 8 - - def test_find_startbit_not_byte_aligned(self): - a = BitStream('0b0010000100') - found = a.find('0b1', start=4) - assert (found, a.bitpos) == ((7,), 7) - found = a.find('0b1', start=2) - assert (found, a.bitpos) == ((2,), 2) - found = a.find('0b1', bytealigned=False, start=8) - assert (found, a.bitpos) == ((), 2) - - def test_find_endbit_not_byte_aligned(self): - a = BitStream('0b0010010000') - found = a.find('0b1', bytealigned=False, end=2) - assert (found, a.bitpos) == ((), 0) - found = a.find('0b1', end=3) - assert (found, a.bitpos) == ((2,), 2) - found = a.find('0b1', bytealigned=False, start=3, end=5) - assert (found, a.bitpos) == ((), 2) - found = a.find('0b1', start=3, end=6) - assert (found[0], a.bitpos) == (5, 5) - - def test_find_startbit_byte_aligned(self): - a = BitStream('0xff001122ff0011ff') - a.pos = 40 - found = a.find('0x22', start=23, bytealigned=True) - assert (found, a.bytepos) == ((24,), 3) - a.bytepos = 4 - found = a.find('0x22', start=24, bytealigned=True) - assert (found, a.bytepos) == ((24,), 3) - found = a.find('0x22', start=25, bytealigned=True) - assert (found, a.pos) == ((), 24) - found = a.find('0b111', start=40, bytealigned=True) - assert (found, a.pos) == ((56,), 56) - - def test_find_endbit_byte_aligned(self): - a = BitStream('0xff001122ff0011ff') - found = a.find('0x22', end=31, bytealigned=True) - assert not found - assert a.pos == 0 - found = a.find('0x22', end=32, bytealigned=True) - assert found - assert a.pos == 24 - assert found[0] == 24 - - def test_find_start_endbit_errors(self): - a = BitStream('0b00100') - with pytest.raises(ValueError): - _ = a.find('0b1', bytealigned=False, start=-100) - with pytest.raises(ValueError): - _ = a.find('0b1', end=6) - with pytest.raises(ValueError): - _ = a.find('0b1', start=4, end=3) - b = BitStream('0x0011223344') - with pytest.raises(ValueError): - _ = b.find('0x22', bytealigned=True, start=-100) - with pytest.raises(ValueError): - _ = b.find('0x22', end=41, bytealigned=True) - - def test_prepend_and_append_again(self): - c = BitStream('0x1122334455667788') - c.bitpos = 40 - c.append('0b1') - assert c.bitpos == len(c) - c = BitStream() - c.prepend('0x1234') - assert c.bytepos == 0 - c = BitStream() - c.append('0x1234') - assert c.bytepos == 2 - s = BitStream(bytes=b'\xff\xff', offset=2) - assert s.length == 14 - t = BitStream(bytes=b'\x80', offset=1, length=2) - s.prepend(t) - assert s == '0x3fff' - - def test_find_all(self): - a = BitStream('0b11111') - p = a.findall('0b1') - assert list(p) == [0, 1, 2, 3, 4] - p = a.findall('0b11') - assert list(p) == [0, 1, 2, 3] - p = a.findall('0b10') - assert list(p) == [] - a = BitStream('0x4733eeff66554747335832434547') - p = a.findall('0x47', bytealigned=True) - assert list(p) == [0, 6 * 8, 7 * 8, 13 * 8] - p = a.findall('0x4733', bytealigned=True) - assert list(p) == [0, 7 * 8] - a = BitStream('0b1001001001001001001') - p = a.findall('0b1001', bytealigned=False) - assert list(p) == [0, 3, 6, 9, 12, 15] - assert a.pos == 0 - - def test_find_all_generator(self): - a = BitStream('0xff1234512345ff1234ff12ff') - p = a.findall('0xff', bytealigned=True) - assert next(p) == 0 - assert next(p) == 6 * 8 - assert next(p) == 9 * 8 - assert next(p) == 11 * 8 - with pytest.raises(StopIteration): - _ = next(p) - - def test_find_all_count(self): - s = BitStream('0b1') * 100 - for i in [0, 1, 23]: - assert len(list(s.findall('0b1', count=i))) == i - with pytest.raises(ValueError): - _ = s.findall('0b1', bytealigned=True, count=-1) - - def test_contains(self): - a = BitStream('0b1') + '0x0001dead0001' - assert '0xdead' in a - assert a.pos == 0 - assert not '0xfeed' in a - - def test_repr(self): - max_ = bitstring.bits.MAX_CHARS - bls = ['', '0b1', '0o5', '0x43412424f41', '0b00101001010101'] - for bs in bls: - a = BitStream(bs) - b = eval(a.__repr__()) - assert a == b - filename = os.path.join(THIS_DIR, 'test.m1v') - for f in [ConstBitStream(filename=filename), - ConstBitStream(filename=filename, length=17), - ConstBitStream(filename=filename, length=23, offset=23102)]: - f2 = eval(f.__repr__()) - assert f2.tobytes() == f.tobytes() - a = BitStream('0b1') - assert repr(a) == "BitStream('0b1')" - a += '0b11' - a.pos = 2 - assert repr(a) == "BitStream('0b111', pos=2)" - a.pos = 0 - a += '0b1' - assert repr(a) == "BitStream('0xf', pos=4)" - a.pos = 0 - a *= max_ - assert repr(a) == "BitStream('0x" + "f" * max_ + "')" - a += '0xf' - assert repr(a) == "BitStream('0x" + "f" * max_ + "...', pos=1004) # length=%d" % (max_ * 4 + 4) - - def test_print(self): - s = BitStream(hex='0x00') - assert '0x' + s.hex == s.__str__() - s = BitStream(filename=os.path.join(THIS_DIR, 'test.m1v')) - assert '0x' + s[0: bitstring.bits.MAX_CHARS * 4].hex + '...' == s.__str__() - assert BitStream().__str__() == '' - s = BitStream('0b11010') - assert '0b' + s.bin == s.__str__() - s = BitStream('0x12345678901234567890,0b1') - assert '0x12345678901234567890, 0b1' == s.__str__() - - def test_iter(self): - a = BitStream('0b001010') - b = BitStream() - for bit in a: - b.append(ConstBitStream(bool=bit)) - assert a == b - - def test_delitem(self): - a = BitStream('0xffee') - del a[0:8] - assert a.hex == 'ee' - del a[0:8] - assert not a - del a[10:12] - assert not a - - def test_non_zero_bits_at_start(self): - a = BitStream(bytes=b'\xff', offset=2) - b = BitStream('0b00') - b += a - assert b == '0b0011 1111' - assert a.tobytes() == b'\xfc' - - def test_non_zero_bits_at_end(self): - a = BitStream(bytes=b'\xff', length=5) - b = BitStream('0b00') - a += b - assert a == '0b1111100' - assert a.tobytes() == b'\xf8' - with pytest.raises(ValueError): - _ = a.bytes - - def test_new_offset_errors(self): - with pytest.raises(bitstring.CreationError): - _ = BitStream(hex='ff', offset=-1) - with pytest.raises(bitstring.CreationError): - _ = BitStream('0xffffffff', offset=33) - - def test_slice_step(self): - a = BitStream('0x3') - b = a[::1] - assert a == b - assert a[2:4:1] == '0b11' - assert a[0:2:1] == '0b00' - assert a[:3] == '0o1' - - a = BitStream('0x0011223344556677') - assert a[-8:] == '0x77' - assert a[:-24] == '0x0011223344' - assert a[-1000:-24] == '0x0011223344' - - def test_interesting_slice_step(self): - a = BitStream('0b0011000111') - assert a[7:3:-1] == '0b1000' - assert a[9:2:-1] == '0b1110001' - assert a[8:2:-2] == '0b100' - assert a[100:-20:-3] == '0b1010' - assert a[100:-20:-1] == '0b1110001100' - assert a[10:2:-1] == '0b1110001' - assert a[100:2:-1] == '0b1110001' - - def test_insertion_order_and_bitpos(self): - b = BitStream() - b[0:0] = '0b0' - b[0:0] = '0b1' - assert b == '0b10' - assert b.bitpos == 0 - a = BitStream() - a.insert('0b0') - a.insert('0b1') - assert a == '0b01' - assert a.bitpos == 2 - - def test_overwrite_order_and_bitpos(self): - a = BitStream('0xff') - a.overwrite('0xa') - assert a == '0xaf' - assert a.bitpos == 4 - a.overwrite('0xb') - assert a == '0xab' - assert a.bitpos == 8 - a.overwrite('0xa', 4) - assert a == '0xaa' - assert a.bitpos == 8 - a.overwrite(a, 0) - assert a == '0xaa' - - def test_init_slice_with_int(self): - a = BitStream(length=8) - a[:] = 100 - assert a.uint == 100 - a[0] = 1 - assert a.bin == '11100100' - a[1] = 0 - assert a.bin == '10100100' - a[-1] = -1 - assert a.bin == '10100101' - a[-3:] = -2 - assert a.bin == '10100110' - - def test_init_slice_with_int_errors(self): - a = BitStream('0b0000') - with pytest.raises(ValueError): - a[0:4] = 16 - with pytest.raises(ValueError): - a[0:4] = -9 - with pytest.raises(ValueError): - a[0] = 2 - with pytest.raises(ValueError): - a[0] = -2 - - def test_reverse_with_slice(self): - a = BitStream('0x0012ff') - a.reverse() - assert a == '0xff4800' - a.reverse(8, 16) - assert a == '0xff1200' - b = a[8:16] - b.reverse() - a[8:16] = b - assert a == '0xff4800' - - def test_reverse_with_slice_errors(self): - a = BitStream('0x123') - with pytest.raises(ValueError): - a.reverse(-1, 4) - with pytest.raises(ValueError): - a.reverse(10, 9) - with pytest.raises(ValueError): - a.reverse(1, 10000) - - def test_initialise_from_list(self): - a = BitStream([]) - assert not a - a = BitStream([True, False, [], [0], 'hello']) - assert a == '0b10011' - a += [] - assert a == '0b10011' - a += [True, False, True] - assert a == '0b10011101' - a.find([12, 23]) - assert a.pos == 3 - assert [1, 0, False, True] == BitStream('0b1001') - a = [True] + BitStream('0b1') - assert a == '0b11' - - def test_initialise_from_tuple(self): - a = BitStream(()) - assert not a - a = BitStream((0, 1, '0', '1')) - assert '0b0111' == a - a.replace((True, True), []) - assert a == (False, True) - - def test_cut(self): - a = BitStream('0x00112233445') - b = list(a.cut(8)) - assert b == ['0x00', '0x11', '0x22', '0x33', '0x44', '0x5'] - b = list(a.cut(4, 8, 16)) - assert b == ['0x1', '0x1'] - b = list(a.cut(4, 0, 44, 4)) - assert b == ['0x0', '0x0', '0x1', '0x1'] - a = BitStream() - b = list(a.cut(10)) - assert not b - - def test_cut_errors(self): - a = BitStream('0b1') - b = a.cut(1, 1, 2) - with pytest.raises(ValueError): - _ = next(b) - b = a.cut(1, -2, 1) - with pytest.raises(ValueError): - _ = next(b) - b = a.cut(0) - with pytest.raises(ValueError): - _ = next(b) - b = a.cut(1, count=-1) - with pytest.raises(ValueError): - _ = next(b) - - def test_cut_problem(self): - s = BitStream('0x1234') - for n in list(s.cut(4)): - s.prepend(n) - assert s == '0x43211234' - - def test_join_functions(self): - a = BitStream().join(['0xa', '0xb', '0b1111']) - assert a == '0xabf' - a = BitStream('0b1').join(['0b0' for _ in range(10)]) - assert a == '0b0101010101010101010' - a = BitStream('0xff').join([]) - assert not a - a = BitStream('0xff').join([Bits(5), '0xab', '0xabc']) - assert a == '0b00000, 0xffabffabc' - - def test_adding_bitpos(self): - a = BitStream('0xff') - b = BitStream('0x00') - a.bitpos = b.bitpos = 8 - c = a + b - assert c.bitpos == 0 - - def test_intelligent_read1(self): - a = BitStream(uint=123, length=23) - u = a.read('uint:23') - assert u == 123 - assert a.pos == a.len - b = BitStream(int=-12, length=44) - i = b.read('int:44') - assert i == -12 - assert b.pos == b.len - u2, i2 = (a + b).readlist('uint:23, int:44') - assert (u2, i2) == (123, -12) - - def test_intelligent_read2(self): - a = BitStream(ue=822) - u = a.read('ue') - assert u == 822 - assert a.pos == a.len - b = BitStream(se=-1001) - s = b.read('se') - assert s == -1001 - assert b.pos == b.len - s, u1, u2 = (b + 2 * a).readlist('se, ue, ue') - assert (s, u1, u2) == (-1001, 822, 822) - - def test_intelligent_read3(self): - a = BitStream('0x123') + '0b11101' - h = a.read('hex:12') - assert h == '123' - b = a.read(' bin : 5 ') - assert b == '11101' - c = '0b' + b + a - b, h = c.readlist('bin:5, hex:12') - assert (b, h) == ('11101', '123') - - def test_intelligent_read4(self): - a = BitStream('0o007') - o = a.read('oct:9') - assert o == '007' - assert a.pos == a.len - - def test_intelligent_read5(self): - a = BitStream('0x00112233') - c0, c1, c2 = a.readlist('bits:8, bits:8, bits:16') - assert (c0, c1, c2) == (BitStream('0x00'), BitStream('0x11'), BitStream('0x2233')) - a.pos = 0 - c = a.read('bits:16') - assert c == BitStream('0x0011') - - def test_intelligent_read6(self): - a = BitStream('0b000111000') - b1, b2, b3 = a.readlist('bin :3, int: 3, int:3') - assert b1 == '000' - assert b2 == -1 - assert b3 == 0 - - def test_intelligent_read7(self): - a = BitStream('0x1234') - a1, a2, a3, a4 = a.readlist('bin:0, oct:0, hex:0, bits:0') - assert a1 == a2 == a3 == '' - assert not a4 - with pytest.raises(ValueError): - _ = a.read('int:0') - with pytest.raises(ValueError): - _ = a.read('uint:0') - assert a.pos == 0 - - def test_intelligent_read8(self): - a = BitStream('0x123456') - for t in ['hex:1', 'oct:1', '-5', 'fred', 'bin:-2', - 'uint:p', 'uint:-2', 'int:u', 'int:-3', 'ses', 'uee', '-14']: - with pytest.raises(ValueError): - _ = a.read(t) - - def test_intelligent_read9(self): - a = BitStream('0xff') - assert a.read('intle8') == -1 - - def test_intelligent_peek(self): - a = BitStream('0b01, 0x43, 0o4, uint:23=2, se=5, ue=3') - b, c, e = a.peeklist('bin:2, hex:8, oct:3') - assert (b, c, e) == ('01', '43', '4') - assert a.pos == 0 - a.pos = 13 - f, g, h = a.peeklist('uint:23, se, ue') - assert (f, g, h) == (2, 5, 3) - assert a.pos == 13 - - def test_read_multiple_bits(self): - s = BitStream('0x123456789abcdef') - a, b = s.readlist([4, 4]) - assert a == '0x1' - assert b == '0x2' - c, d, e = s.readlist([8, 16, 8]) - assert c == '0x34' - assert d == '0x5678' - assert e == '0x9a' - - def test_peek_multiple_bits(self): - s = BitStream('0b1101, 0o721, 0x2234567') - a, b, c, d = s.peeklist([2, 1, 1, 9]) - assert a == '0b11' - assert bool(b) == True - assert bool(c) == True - assert d == '0o721' - assert s.pos == 0 - a, b = s.peeklist([4, 9]) - assert a == '0b1101' - assert b == '0o721' - s.pos = 13 - a, b = s.peeklist([16, 8]) - assert a == '0x2234' - assert b == '0x56' - assert s.pos == 13 - - def test_difficult_prepends(self): - a = BitStream('0b1101011') - b = BitStream() - for i in range(10): - b.prepend(a) - assert b == a * 10 - - def test_packing_wrong_number_of_things(self): - with pytest.raises(bitstring.CreationError): - _ = pack('bin:1') - with pytest.raises(bitstring.CreationError): - _ = pack('', 100) - - def test_pack_with_various_keys(self): - a = pack('uint10', uint10='0b1') - assert a == '0b1' - b = pack('0b110', **{'0b110': '0xfff'}) - assert b == '0xfff' - - def test_pack_with_variable_length(self): - for i in range(1, 11): - a = pack('uint:n', 0, n=i) - assert a.bin == '0' * i - - def test_to_bytes(self): - a = BitStream(bytes=b'\xab\x00') - b = a.tobytes() - assert a.bytes == b - for i in range(7): - del a[-1:] - assert a.tobytes() == b'\xab\x00' - del a[-1:] - assert a.tobytes() == b'\xab' - - def test_to_file(self): - filename = os.path.join(THIS_DIR, 'temp_bitstring_unit_testing_file') - a = BitStream('0x0000ff')[:17] - with open(filename, 'wb') as f: - a.tofile(f) - b = BitStream(filename=filename) - assert b == '0x000080' - - a = BitStream('int:1000000=-1') - assert a.int == -1 - with open(filename, 'wb') as f: - a.tofile(f) - b = BitStream(filename=filename) - assert b.int == -1 - assert b.len == 1000000 - - def test_token_parser(self): - tp = bitstring.utils.tokenparser - assert tp('hex') == (True, [('hex', None, None)]) - assert tp('hex=14') == (True, [('hex', None, '14')]) - assert tp('0xef') == (False, [('0x', None, 'ef')]) - assert tp('uint:12') == (False, [('uint', 12, None)]) - assert tp('int:30=-1') == (False, [('int', 30, '-1')]) - assert tp('bits10') == (False, [('bits', 10, None)]) - assert tp('bits:10') == (False, [('bits', 10, None)]) - assert tp('123') == (False, [('bits', 123, None)]) - assert tp('123') == (False, [('bits', 123, None)]) - assert tp('hex12', ('hex12',)) == (False, [('hex12', None, None)]) - assert tp('2*bits:6') == (False, [('bits', 6, None), ('bits', 6, None)]) - - def test_token_parser_struct_codes(self): - tp = bitstring.utils.tokenparser - assert tp('>H') == (False, [('uintbe', 16, None)]) - assert tp('b') == (False, [('int', 8, None)]) - assert tp('b', 23) == BitStream('intbe:8=23') - assert pack('>B', 23) == BitStream('uintbe:8=23') - assert pack('>h', 23) == BitStream('intbe:16=23') - assert pack('>H', 23) == BitStream('uintbe:16=23') - assert pack('>l', 23) == BitStream('intbe:32=23') - assert pack('>L', 23) == BitStream('uintbe:32=23') - assert pack('>i', 23) == BitStream('intbe:32=23') - assert pack('>I', 23) == BitStream('uintbe:32=23') - assert pack('>q', 23) == BitStream('intbe:64=23') - assert pack('>Q', 23) == BitStream('uintbe:64=23') - with pytest.raises(bitstring.CreationError): - _ = pack('2i', 40, 40) - - def test_struct_tokens3(self): - s = pack('>hhl', 1, 2, 3) - a, b, c = s.unpack('>hhl') - assert (a, b, c) == (1, 2, 3) - s = pack('Q \tL', 1001, 43, 21, 9999) - assert s.unpack('QL') == [1001, 43, 21, 9999] - - def test_struct_tokens_multiplicative_factors(self): - s = pack('<2h', 1, 2) - a, b = s.unpack('<2h') - assert (a, b) == (1, 2) - s = pack('<100q', *range(100)) - assert s.len == 100 * 64 - assert s[44*64:45*64].uintle == 44 - s = pack('@L0B2h', 5, 5, 5) - assert s.unpack('@Lhh') == [5, 5, 5] - - def test_struct_tokens_errors(self): - for f in ['>>q', '<>q', 'q>', '2q', 'q', '>-2q', '@a', '>int:8', '>q2']: - with pytest.raises(bitstring.CreationError): - _ = pack(f, 100) - - def test_immutable_bit_streams(self): - a = ConstBitStream('0x012345') - assert a == '0x012345' - b = BitStream('0xf') + a - assert b == '0xf012345' - with pytest.raises(AttributeError): - a.append(b) - with pytest.raises(AttributeError): - a.prepend(b) - with pytest.raises(TypeError): - a[0] = '0b1' - with pytest.raises(TypeError): - del a[5] - with pytest.raises(AttributeError): - a.replace('0b1', '0b0') - with pytest.raises(AttributeError): - a.insert('0b11', 4) - with pytest.raises(AttributeError): - a.reverse() - with pytest.raises(AttributeError): - a.reversebytes() - assert a == '0x012345' - assert isinstance(a, ConstBitStream) - - def test_reverse_bytes(self): - a = BitStream('0x123456') - a.byteswap() - assert a == '0x563412' - b = a + '0b1' - b.byteswap() - assert '0x123456, 0b1' == b - a = BitStream('0x54') - a.byteswap() - assert a == '0x54' - a = BitStream() - a.byteswap() - assert not a - - def test_reverse_bytes2(self): - a = BitStream() - a.byteswap() - assert not a - a = BitStream('0x00112233') - a.byteswap(0, 0, 16) - assert a == '0x11002233' - a.byteswap(0, 4, 28) - assert a == '0x12302103' - a.byteswap(start=0, end=18) - assert a == '0x30122103' - with pytest.raises(ValueError): - a.byteswap(0, 10, 2) - with pytest.raises(ValueError): - a.byteswap(0, -4, 4) - with pytest.raises(ValueError): - a.byteswap(0, 24, 48) - a.byteswap(0, 24) - assert a == '0x30122103' - a.byteswap(0, 11, 11) - assert a == '0x30122103' - - def test_capitals_in_pack(self): - a = pack('A', A='0b1') - assert a == '0b1' - format_ = 'bits:4=BL_OFFT, uint:12=width, uint:12=height' - d = {'BL_OFFT': '0b1011', 'width': 352, 'height': 288} - s = bitstring.pack(format_, **d) - assert s == '0b1011, uint:12=352, uint:12=288' - a = pack('0X0, uint:8, hex', 45, '0XABcD') - assert a == '0x0, uint:8=45, 0xabCD' - - def test_other_capitals(self): - a = ConstBitStream('0XABC, 0O0, 0B11') - assert a == 'hex=0Xabc, oct=0, bin=0B11' - - def test_efficient_overwrite(self): - a = BitStream(100000000) - a.overwrite([1], 123456) - assert a[123456] == True - a.overwrite('0xff', 1) - assert a[0:32:1] == '0x7f800000' - b = BitStream('0xffff') - b.overwrite('0x0000') - assert b == '0x0000' - assert b.pos == 16 - c = BitStream(length=1000) - c.overwrite('0xaaaaaaaaaaaa', 81) - assert c[81:81 + 6 * 8] == '0xaaaaaaaaaaaa' - assert len(list(c.findall('0b1'))) == 24 - s = BitStream(length=1000) - s = s[5:] - s.overwrite('0xffffff', 500) - s.pos = 500 - assert s.read(4 * 8) == '0xffffff00' - s.overwrite('0xff', 502) - assert s[502:518] == '0xffff' - - def test_peek_and_read_list_errors(self): - a = BitStream('0x123456') - with pytest.raises(ValueError): - _ = a.read('hex:8, hex:8') - with pytest.raises(ValueError): - _ = a.peek('hex:8, hex:8') - with pytest.raises(TypeError): - _ = a.read(10, 12) - with pytest.raises(TypeError): - _ = a.peek(12, 14) - with pytest.raises(TypeError): - _ = a.read(8, 8) - with pytest.raises(TypeError): - _ = a.peek(80, 80) - - def test_startswith(self): - a = BitStream() - assert a.startswith(BitStream()) - assert not a.startswith('0b0') - a = BitStream('0x12ff') - assert a.startswith('0x1') - assert a.startswith('0b0001001') - assert a.startswith('0x12ff') - assert not a.startswith('0x12ff, 0b1') - assert not a.startswith('0x2') - - def test_startswith_start_end(self): - s = BitStream('0x123456') - assert s.startswith('0x234', 4) - assert not s.startswith('0x123', end=11) - assert s.startswith('0x123', end=12) - assert s.startswith('0x34', 8, 16) - assert not s.startswith('0x34', 7, 16) - assert not s.startswith('0x34', 9, 16) - assert not s.startswith('0x34', 8, 15) - - def test_endswith(self): - a = BitStream() - assert a.endswith('') - assert not a.endswith(BitStream('0b1')) - a = BitStream('0xf2341') - assert a.endswith('0x41') - assert a.endswith('0b001') - assert a.endswith('0xf2341') - assert not a.endswith('0x1f2341') - assert not a.endswith('0o34') - - def test_endswith_start_end(self): - s = BitStream('0x123456') - assert s.endswith('0x234', end=16) - assert not s.endswith('0x456', start=13) - assert s.endswith('0x456', start=12) - assert s.endswith('0x34', 8, 16) - assert s.endswith('0x34', 7, 16) - assert not s.endswith('0x34', 9, 16) - assert not s.endswith('0x34', 8, 15) - - def test_unhashability(self): - s = BitStream('0xf') - with pytest.raises(TypeError): - _ = {s} - with pytest.raises(TypeError): - _ = hash([s]) - - def test_const_bit_stream_set_creation(self): - sl = [ConstBitStream(uint=i, length=7) for i in range(15)] - s = set(sl) - assert len(s) == 15 - s.add(ConstBitStream('0b0000011')) - assert len(s) == 15 - with pytest.raises(TypeError): - s.add(BitStream('0b0000011')) - - def test_const_bit_stream_functions(self): - s = ConstBitStream('0xf, 0b1') - assert type(s) == ConstBitStream - t = copy.copy(s) - assert type(t) == ConstBitStream - a = s + '0o3' - assert type(a) == ConstBitStream - b = a[0:4] - assert type(b) == ConstBitStream - b = a[4:3] - assert type(b) == ConstBitStream - b = a[5:2:-1] - assert type(b) == ConstBitStream - b = ~a - assert type(b) == ConstBitStream - b = a << 2 - assert type(b) == ConstBitStream - b = a >> 2 - assert type(b) == ConstBitStream - b = a * 2 - assert type(b) == ConstBitStream - b = a * 0 - assert type(b) == ConstBitStream - b = a & ~a - assert type(b) == ConstBitStream - b = a | ~a - assert type(b) == ConstBitStream - b = a ^ ~a - assert type(b) == ConstBitStream - b = a._slice(4, 4) - assert type(b) == ConstBitStream - b = a.read(4) - assert type(b) == ConstBitStream - - def test_const_bit_stream_properties(self): - a = ConstBitStream('0x123123') - with pytest.raises(AttributeError): - a.hex = '0x234' - with pytest.raises(AttributeError): - a.oct = '0o234' - with pytest.raises(AttributeError): - a.bin = '0b101' - with pytest.raises(AttributeError): - a.ue = 3453 - with pytest.raises(AttributeError): - a.se = -123 - with pytest.raises(AttributeError): - a.int = 432 - with pytest.raises(AttributeError): - a.uint = 4412 - with pytest.raises(AttributeError): - a.intle = 123 - with pytest.raises(AttributeError): - a.uintle = 4412 - with pytest.raises(AttributeError): - a.intbe = 123 - with pytest.raises(AttributeError): - a.uintbe = 4412 - with pytest.raises(AttributeError): - a.intne = 123 - with pytest.raises(AttributeError): - a.uintne = 4412 - with pytest.raises(AttributeError): - a.bytes = b'hello' - - def test_const_bit_stream_misc(self): - a = ConstBitStream('0xf') - b = a - a += '0xe' - assert b == '0xf' - assert a == '0xfe' - c = BitStream(a) - assert a == c - a = ConstBitStream('0b1') - a += a - assert a == '0b11' - assert type(a) == ConstBitStream - a._addleft(a) - assert a == '0b1111' - assert type(a) == ConstBitStream - - def test_const_bit_stream_hashibility(self): - a = ConstBitStream('0x1') - b = ConstBitStream('0x2') - c = ConstBitStream('0x1') - c.pos = 3 - s = {a, b, c} - assert len(s) == 2 - assert hash(a) == hash(c) - - def test_const_hashability_again(self): - a = ConstBitStream(uint=1 << 300, length=10000) - b = ConstBitStream(uint=2 << 300, length=10000) - c = ConstBitStream(uint=3 << 300, length=10000) - s = {a, b, c} - assert len(s) == 3 - - def test_hash_edge_cases(self): - a = ConstBitStream('0xabcd') - b = ConstBitStream('0xabcd') - c = b[1:] - assert hash(a) == hash(b) - assert hash(a) != hash(c) - - def test_const_bit_stream_copy(self): - a = ConstBitStream('0xabc') - a.pos = 11 - b = copy.copy(a) - b.pos = 4 - assert id(a._bitstore) == id(b._bitstore) - assert a.pos == 11 - assert b.pos == 4 - - def test_python26stuff(self): - s = BitStream('0xff') - assert isinstance(s.tobytes(), bytes) - assert isinstance(s.bytes, bytes) - - def test_read_from_bits(self): - a = ConstBitStream('0xaabbccdd') - b = a.read(8) - assert b == '0xaa' - assert a[0:8] == '0xaa' - assert a[-1] == True - a.pos = 0 - assert a.read(4).uint == 10 - - -class TestSet: - def test_set(self): - a = BitStream(length=16) - a.set(True, 0) - assert a == '0b10000000 00000000' - a.set(1, 15) - assert a == '0b10000000 00000001' - b = a[4:12] - b.set(True, 1) - assert b == '0b01000000' - b.set(True, -1) - assert b == '0b01000001' - b.set(1, -8) - assert b == '0b11000001' - with pytest.raises(IndexError): - b.set(True, -9) - with pytest.raises(IndexError): - b.set(True, 8) - - def test_set_negative_index(self): - a = BitStream(10) - a.set(1, -1) - assert a.bin == '0000000001' - a.set(1, [-1, -10]) - assert a.bin == '1000000001' - with pytest.raises(IndexError): - a.set(1, [-11]) - - def test_file_based_set_unset(self): - filename = os.path.join(THIS_DIR, 'test.m1v') - a = BitStream(filename=filename) - a.set(True, (0, 1, 2, 3, 4)) - assert a[0:32] == '0xf80001b3' - a = BitStream(filename=filename) - a.set(False, (28, 29, 30, 31)) - assert a.startswith('0x000001b0') - - def test_set_list(self): - a = BitStream(length=18) - a.set(True, range(18)) - assert a.int == -1 - a.set(False, range(18)) - assert a.int == 0 - - def test_unset(self): - a = BitStream(length=16, int=-1) - a.set(False, 0) - assert ~a == '0b10000000 00000000' - a.set(0, 15) - assert ~a == '0b10000000 00000001' - b = a[4:12] - b.set(False, 1) - assert ~b == '0b01000000' - b.set(False, -1) - assert ~b == '0b01000001' - b.set(False, -8) - assert ~b == '0b11000001' - with pytest.raises(IndexError): - b.set(False, -9) - with pytest.raises(IndexError): - b.set(False, 8) - - def test_set_whole_bit_stream(self): - a = BitStream(10000) - a.set(1) - assert a.all(1) - a.set(0) - assert a.all(0) - - -class TestInvert: - def test_invert_bits(self): - a = BitStream('0b111000') - a.invert(range(a.len)) - assert a == '0b000111' - a.invert([0, 1, -1]) - assert a == '0b110110' - - def test_invert_whole_bit_stream(self): - a = BitStream('0b11011') - a.invert() - assert a == '0b00100' - - def test_invert_single_bit(self): - a = BitStream('0b000001') - a.invert(0) - assert a.bin == '100001' - a.invert(-1) - assert a.bin == '100000' - - def test_invert_errors(self): - a = BitStream(10) - with pytest.raises(IndexError): - a.invert(10) - with pytest.raises(IndexError): - a.invert(-11) - with pytest.raises(IndexError): - a.invert([1, 2, 10]) - - def test_ior(self): - a = BitStream('0b1101001') - a |= '0b1110000' - assert a == '0b1111001' - b = a[2:] - c = a[1:-1] - b |= c - assert c == '0b11100' - assert b == '0b11101' - - def test_iand(self): - a = BitStream('0b0101010101000') - a &= '0b1111110000000' - assert a == '0b0101010000000' - s = BitStream(filename=os.path.join(THIS_DIR, 'test.m1v'), offset=26, length=24) - s &= '0xff00ff' - assert s == '0xcc0004' - - def test_ixor(self): - a = BitStream('0b11001100110011') - a ^= '0b11111100000010' - assert a == '0b00110000110001' - - def test_logical_inplace_errors(self): - a = BitStream(4) - with pytest.raises(ValueError): - a |= '0b111' - with pytest.raises(ValueError): - a &= '0b111' - with pytest.raises(ValueError): - a ^= '0b111' - - -class TestAllAndAny: - def test_all(self): - a = BitStream('0b0111') - assert a.all(True, (1, 3)) - assert not a.all(True, (0, 1, 2)) - assert a.all(True, [-1]) - assert not a.all(True, [0]) - - def test_file_based_all(self): - filename = os.path.join(THIS_DIR, 'test.m1v') - a = BitStream(filename=filename) - assert a.all(True, [31]) - a = BitStream(filename=filename) - assert a.all(False, (0, 1, 2, 3, 4)) - - def test_file_based_any(self): - filename = os.path.join(THIS_DIR, 'test.m1v') - a = BitStream(filename=filename) - assert a.any(True, (31, 12)) - a = BitStream(filename=filename) - assert a.any(False, (0, 1, 2, 3, 4)) - b = ConstBitStream(filename=filename, offset=16) - assert b.startswith('0x01') - assert not b.any(True, range(0, 7)) - assert b.any(True, range(0, 8)) - assert b.any(True) - - def test_any(self): - a = BitStream('0b10011011') - assert a.any(True, (1, 2, 3, 5)) - assert not a.any(True, (1, 2, 5)) - assert a.any(True, (-1,)) - assert not a.any(True, (1,)) - - def test_all_false(self): - a = BitStream('0b0010011101') - assert a.all(False, (0, 1, 3, 4)) - assert not a.all(False, (0, 1, 2, 3, 4)) - - def test_any_false(self): - a = BitStream('0b01001110110111111111111111111') - assert a.any(False, (4, 5, 6, 2)) - assert not a.any(False, (1, 15, 20)) - - def test_any_empty_bitstring(self): - a = ConstBitStream() - assert not a.any(True) - assert not a.any(False) - - def test_all_empty_bit_stream(self): - a = ConstBitStream() - assert a.all(True) - assert a.all(False) - - def test_any_whole_bitstring(self): - a = ConstBitStream('0xfff') - assert a.any(True) - assert not a.any(False) - - def test_all_whole_bitstring(self): - a = ConstBitStream('0xfff') - assert a.all(True) - assert not a.all(False) - - def test_errors(self): - a = BitStream('0xf') - with pytest.raises(IndexError): - a.all(True, [5]) - with pytest.raises(IndexError): - a.all(True, [-5]) - with pytest.raises(IndexError): - a.any(True, [5]) - with pytest.raises(IndexError): - a.any(True, [-5]) - - ################### - - def test_float_initialisation(self): - for f in (0.000001, -1.0, 1.0, 0.2, -3.14159265): - a = BitStream(float=f, length=64) - a.pos = 6 - assert a.float == f - a = BitStream('float:64=%s' % str(f)) - a.pos = 6 - assert a.float == f - a = BitStream('floatbe:64=%s' % str(f)) - a.pos = 6 - assert a.floatbe == f - a = BitStream('floatle:64=%s' % str(f)) - a.pos = 6 - assert a.floatle == f - a = BitStream('floatne:64=%s' % str(f)) - a.pos = 6 - assert a.floatne == f - - b = BitStream(float=f, length=32) - b.pos = 6 - assert b.float / f == pytest.approx(1.0) - b = BitStream('float:32=%s' % str(f)) - b.pos = 6 - assert b.float / f == pytest.approx(1.0) - b = BitStream('floatbe:32=%s' % str(f)) - b.pos = 6 - assert b.floatbe / f == pytest.approx(1.0) - b = BitStream('floatle:32=%s' % str(f)) - b.pos = 6 - assert b.floatle / f == pytest.approx(1.0) - b = BitStream('floatne:32=%s' % str(f)) - b.pos = 6 - assert b.floatne / f == pytest.approx(1.0) - - a = BitStream(float=f, length=16) - a.pos = 6 - assert a.float == pytest.approx(f, abs=0.01) - a = BitStream('float:16=%s' % str(f)) - a.pos = 6 - assert a.float == pytest.approx(f, abs=0.01) - a = BitStream('floatbe:16=%s' % str(f)) - a.pos = 6 - assert a.floatbe == pytest.approx(f, abs=0.01) - a = BitStream('floatle:16=%s' % str(f)) - a.pos = 6 - assert a.floatle == pytest.approx(f, abs=0.01) - a = BitStream('floatne:16=%s' % str(f)) - a.pos = 6 - assert a.floatne == pytest.approx(f, abs=0.01) - - a = BitStream('0x12345678') - a.pos = 6 - a.f = 23 - assert a.f == 23.0 - - def test_float_init_strings(self): - for s in ('5', '+0.0001', '-1e101', '4.', '.2', '-.65', '43.21E+32'): - a = BitStream('float:64=%s' % s) - assert a.float == float(s) - for s in ('5', '+0.5', '-1e2', '4.', '.25', '-.75'): - a = BitStream('float:16=%s' % s) - assert a.f == float(s) - - def test_float_packing(self): - a = pack('>d', 0.01) - assert a.float == 0.01 - assert a.floatbe == 0.01 - a.byteswap() - assert a.floatle == 0.01 - b = pack('>f', 1e10) - assert b.float / 1e10 == pytest.approx(1.0) - c = pack('5d', 10.0, 5.0, 2.5, 1.25, 0.1) - assert d.unpack('>5d') == [10.0, 5.0, 2.5, 1.25, 0.1] - e = pack('>3e', -100, 100, 0.25) - assert e.unpack('>3e') == [-100.0, 100.0, 0.25] - - def test_float_reading(self): - a = BitStream('floatle:64=12, floatbe:64=-0.01, floatne:64=3e33') - x, y, z = a.readlist('floatle:64, floatbe:64, floatne:64') - assert x == 12.0 - assert y == -0.01 - assert z == 3e33 - a = BitStream('floatle:16=12, floatbe:32=-0.01, floatne:32=3e33') - x, y, z = a.readlist('floatle:16, floatbe:32, floatne:32') - assert x / 12.0 == pytest.approx(1.0) - assert y / -0.01 == pytest.approx(1.0) - assert z / 3e33 == pytest.approx(1.0) - a = BitStream('0b11, floatle:64=12, 0xfffff') - a.pos = 2 - floatle64 = Dtype('floatle64') - assert a.read(floatle64) == 12.0 - b = BitStream(floatle=20, length=32) - b.floatle = 10.0 - b = [0] + b - assert b[1:].floatle == 10.0 - - def test_non_aligned_float_reading(self): - s = BitStream('0b1, float:32 = 10.0') - x, y = s.readlist('1, float:32') - assert y == 10.0 - s[1:] = 'floatle:32=20.0' - x, y = s.unpack('1, floatle:32') - assert y == 20.0 - - def test_float_errors(self): - a = BitStream('0x3') - with pytest.raises(bitstring.InterpretError): - _ = a.float - with pytest.raises(bitstring.CreationError): - a.float = -0.2 - for le in (8, 10, 12, 18, 30, 128, 200): - with pytest.raises(ValueError): - _ = BitStream(float=1.0, length=le) - with pytest.raises(bitstring.CreationError): - _ = BitStream(floatle=0.3, length=0) - with pytest.raises(bitstring.CreationError): - _ = BitStream(floatle=0.3, length=1) - with pytest.raises(bitstring.CreationError): - _ = BitStream(float=2) - with pytest.raises(bitstring.InterpretError): - _ = a.read('floatle:2') - - def test_read_error_changes_pos(self): - a = BitStream('0x123123') - with pytest.raises(ValueError): - a.read('10, 5') - - def test_ror(self): - a = BitStream('0b11001') - a.ror(0) - assert a == '0b11001' - a.ror(1) - assert a == '0b11100' - a.ror(5) - assert a == '0b11100' - a.ror(101) - assert a == '0b01110' - a = BitStream('0b1') - a.ror(1000000) - assert a == '0b1' - - def test_ror_errors(self): - a = BitStream() - with pytest.raises(bitstring.Error): - a.ror(0) - a += '0b001' - with pytest.raises(ValueError): - a.ror(-1) - - def test_rol(self): - a = BitStream('0b11001') - a.rol(0) - assert a == '0b11001' - a.rol(1) - assert a == '0b10011' - a.rol(5) - assert a == '0b10011' - a.rol(101) - assert a == '0b00111' - a = BitStream('0b1') - a.rol(1000000) - assert a == '0b1' - - def test_rol_from_file(self): - a = BitStream(filename=os.path.join(THIS_DIR, 'test.m1v')) - m = a.len - a.rol(1) - assert a.startswith('0x000003') - assert a.len == m - assert a.endswith('0x0036e') - - def test_ror_from_file(self): - a = BitStream(filename=os.path.join(THIS_DIR, 'test.m1v')) - m = a.len - a.ror(1) - assert a.startswith('0x800000') - assert a.len == m - assert a.endswith('0x000db') - - def test_rol_errors(self): - a = BitStream() - with pytest.raises(bitstring.Error): - a.rol(0) - a += '0b001' - with pytest.raises(ValueError): - a.rol(-1) - - def test_bytes_token(self): - a = BitStream('0x510203') - b = a.read('bytes:1') - assert isinstance(b, bytes) - assert b == b'\x51' - x, y, z = a.unpack('uint:4, bytes:2, uint') - assert x == 5 - assert y == b'\x10\x20' - assert z == 3 - s = pack('bytes:4', b'abcd') - assert s.bytes == b'abcd' - - def test_bytes_token_more_thoroughly(self): - a = BitStream('0x0123456789abcdef') - a.pos += 16 - assert a.read('bytes:1') == b'\x45' - assert a.read('bytes:3') == b'\x67\x89\xab' - x, y, z = a.unpack('bits:28, bytes, bits:12') - assert y == b'\x78\x9a\xbc' - - def test_dedicated_read_functions(self): - a = BitStream('0b11, uint:43=98798798172, 0b11111') - x = a[2:45].uint - assert x == 98798798172 - assert a.pos == 0 - a.pos = 2 - x = a.read(Dtype('int43')) - assert x == 98798798172 - assert a.pos == 45 - - a = BitStream('0b11, uintbe:48=98798798172, 0b11111') - a.pos = 2 - x = a.read(Dtype('uintbe48')) - assert x == 98798798172 - assert a.pos == 50 - - a = BitStream('0b111, uintle:40=123516, 0b111') - a.pos = 3 - assert a.read('uintle:40') == 123516 - b = BitStream('0xff, uintle:800=999, 0xffff') - assert b[8:800].uintle == 999 - - a = BitStream('0b111, intle:48=999999999, 0b111111111111') - a.pos = 3 - assert a.read('intle48') == 999999999 - b = BitStream('0xff, intle:200=918019283740918263512351235, 0xfffffff') - b.pos = 8 - assert b.read(Dtype('intle', length=200)) == 918019283740918263512351235 - - a = BitStream('0b111, bfloat:16=-5.25, 0xffffffff') - a.pos = 3 - assert a.read('bfloatbe') == -5.25 - - a = BitStream('0b111, floatle:64=9.9998, 0b111') - a.pos = 3 - assert a.read('floatle64') == 9.9998 - - def test_auto_init_with_int(self): - a = BitStream(0) - assert not a - a = BitStream(1) - assert a == '0b0' - a = BitStream(1007) - assert a == BitStream(length=1007) - with pytest.raises(bitstring.CreationError): - _ = BitStream(-1) - - assert ConstBitStream(13) == Bits(13) - with pytest.raises(TypeError): - a += 10 - - def test_reading_problems(self): - a = BitStream('0x000001') - b = a.read('uint:24') - assert b == 1 - a.pos = 0 - with pytest.raises(bitstring.ReadError): - _ = a.read('bytes:4') - - @pytest.mark.skip("Bug #266") - def test_pos_reset_bug(self): - a = BitStream('0x0120310230123', pos=23) - assert a.pos == 23 - a.u8 = 14 - assert a.pos == 0 - a.pos = 5 - a.u8 = 9 - assert a.pos == 0 - - def test_creation_exception_bug(self): - with pytest.raises(ValueError): - _ = BitStream(bin=1) - - def test_add_verses_in_place_add(self): - a1 = ConstBitStream('0xabc') - b1 = a1 - a1 += '0xdef' - assert a1 == '0xabcdef' - assert b1 == '0xabc' - - a2 = BitStream('0xabc') - b2 = a2 - c2 = a2 + '0x0' - a2 += '0xdef' - assert a2 == '0xabcdef' - assert b2 == '0xabcdef' - assert c2 == '0xabc0' - - def test_and_verses_in_place_and(self): - a1 = ConstBitStream('0xabc') - b1 = a1 - a1 &= '0xf0f' - assert a1 == '0xa0c' - assert b1 == '0xabc' - - a2 = BitStream('0xabc') - b2 = a2 - c2 = a2 & '0x00f' - a2 &= '0xf0f' - assert a2 == '0xa0c' - assert b2 == '0xa0c' - assert c2 == '0x00c' - - def test_or_verses_in_place_or(self): - a1 = ConstBitStream('0xabc') - b1 = a1 - a1 |= '0xf0f' - assert a1 == '0xfbf' - assert b1 == '0xabc' - - a2 = BitStream('0xabc') - b2 = a2 - c2 = a2 | '0x00f' - a2 |= '0xf0f' - assert a2 == '0xfbf' - assert b2 == '0xfbf' - assert c2 == '0xabf' - - def test_xor_verses_in_place_xor(self): - a1 = ConstBitStream('0xabc') - b1 = a1 - a1 ^= '0xf0f' - assert a1 == '0x5b3' - assert b1 == '0xabc' - - a2 = BitStream('0xabc') - b2 = a2 - c2 = a2 ^ '0x00f' - a2 ^= '0xf0f' - assert a2 == '0x5b3' - assert b2 == '0x5b3' - assert c2 == '0xab3' - - def test_mul_verses_in_place_mul(self): - a1 = ConstBitStream('0xabc') - b1 = a1 - a1 *= 3 - assert a1 == '0xabcabcabc' - assert b1 == '0xabc' - - a2 = BitStream('0xabc') - b2 = a2 - c2 = a2 * 2 - a2 *= 3 - assert a2 == '0xabcabcabc' - assert b2 == '0xabcabcabc' - assert c2 == '0xabcabc' - - def test_lshift_verses_in_place_lshift(self): - a1 = ConstBitStream('0xabc') - b1 = a1 - a1 <<= 4 - assert a1 == '0xbc0' - assert b1 == '0xabc' - - a2 = BitStream('0xabc') - b2 = a2 - c2 = a2 << 8 - a2 <<= 4 - assert a2 == '0xbc0' - assert b2 == '0xbc0' - assert c2 == '0xc00' - - def test_rshift_verses_in_place_rshift(self): - a1 = ConstBitStream('0xabc') - b1 = a1 - a1 >>= 4 - assert a1 == '0x0ab' - assert b1 == '0xabc' - - a2 = BitStream('0xabc') - b2 = a2 - c2 = a2 >> 8 - a2 >>= 4 - assert a2 == '0x0ab' - assert b2 == '0x0ab' - assert c2 == '0x00a' - - def test_auto_from_bool(self): - with pytest.raises(TypeError): - a = ConstBitStream() + True + False + True - - -class TestBugs: - def test_bug_in_replace(self): - s = BitStream('0x00112233') - li = list(s.split('0x22', start=8, bytealigned=True)) - assert li == ['0x11', '0x2233'] - s = BitStream('0x00112233') - s.replace('0x22', '0xffff', start=8, bytealigned=True) - assert s == '0x0011ffff33' - s = BitStream('0x0123412341234') - s.replace('0x23', '0xf', start=9, bytealigned=True) - assert s == '0x012341f41f4' - - def test_null_bits(self): - s = ConstBitStream(bin='') - t = ConstBitStream(oct='') - u = ConstBitStream(hex='') - v = ConstBitStream(bytes=b'') - assert not s - assert not t - assert not u - assert not v - - def test_multiplicative_factors_creation(self): - s = BitStream('1*0b1') - assert s == '0b1' - s = BitStream('4*0xc') - assert s == '0xcccc' - s = BitStream('0b1, 0*0b0') - assert s == '0b1' - s = BitStream('0b1, 3*uint:8=34, 2*0o755') - assert s == '0b1, uint:8=34, uint:8=34, uint:8=34, 0o755755' - s = BitStream('0*0b1001010') - assert not s - - def test_multiplicative_factors_reading(self): - s = BitStream('0xc') * 5 - a, b, c, d, e = s.readlist('5*uint:4') - assert a == b == c == d == e == 12 - s = ConstBitStream('2*0b101, 4*uint:7=3') - a, b, c, d, e = s.readlist('2*bin:3, 3*uint:7') - assert a == b == '101' - assert c == d == e == 3 - - def test_multiplicative_factors_packing(self): - s = pack('3*bin', '1', '001', '101') - assert s == '0b1001101' - s = pack('hex, 2*se=-56, 3*uint:37', '34', 1, 2, 3) - a, b, c, d, e, f = s.unpack('hex:8, 2*se, 3*uint:37') - assert a == '34' - assert b == -56 - assert c == -56 - assert (d, e, f) == (1, 2, 3) - - def test_multiplicative_factors_unpacking(self): - s = ConstBitStream('0b10111') - a, b, c, d = s.unpack('3*bool, bin') - assert (a, b, c) == (True, False, True) - assert d == '11' - - def test_packing_default_int_with_keyword(self): - s = pack('uint:12', 100) - assert s.unpack('12')[0].uint == 100 - s = pack('int:oh_no_not_the_eyes=33', oh_no_not_the_eyes=17) - assert s.int == 33 - assert s.len == 17 - - def test_init_from_iterable(self): - assert isinstance(range(10), collections.abc.Iterable) - s = ConstBitStream(range(12)) - assert s == '0x7ff' - - def test_function_negative_indices(self): - # insert - s = BitStream('0b0111') - s.insert('0b0', -1) - assert s == '0b01101' - with pytest.raises(ValueError): - s.insert('0b0', -1000) - - # reverse - s.reverse(-2) - assert s == '0b01110' - t = BitStream('0x778899abcdef') - t.reverse(-12, -4) - assert t == '0x778899abc7bf' - - # reversebytes - t.byteswap(0, -40, -16) - assert t == '0x77ab9988c7bf' - - # overwrite - t.overwrite('0x666', -20) - assert t == '0x77ab998666bf' - - # find - found = t.find('0x998', bytealigned=True, start=-31) - assert not found - found = t.find('0x998', bytealigned=True, start=-32) - assert found - assert t.pos == 16 - t.pos = 0 - found = t.find('0x988', bytealigned=True, end=-21) - assert not found - found = t.find('0x998', bytealigned=True, end=-20) - assert found - assert t.pos == 16 - - # findall - s = BitStream('0x1234151f') - li = list(s.findall('0x1', bytealigned=True, start=-15)) - assert li == [24] - li = list(s.findall('0x1', bytealigned=True, start=-16)) - assert li == [16, 24] - li = list(s.findall('0x1', bytealigned=True, end=-5)) - assert li == [0, 16] - li = list(s.findall('0x1', bytealigned=True, end=-4)) - assert li == [0, 16, 24] - - # rfind - found = s.rfind('0x1f', end=-1) - assert not found - found = s.rfind('0x12', start=-31) - assert not found - - # cut - s = BitStream('0x12345') - li = list(s.cut(4, start=-12, end=-4)) - assert li == ['0x3', '0x4'] - - # split - s = BitStream('0xfe0012fe1200fe') - li = list(s.split('0xfe', bytealigned=True, end=-1)) - assert li == ['', '0xfe0012', '0xfe1200f, 0b111'] - li = list(s.split('0xfe', bytealigned=True, start=-8)) - assert li == ['', '0xfe'] - - # startswith - assert s.startswith('0x00f', start=-16) - assert s.startswith('0xfe00', end=-40) - assert not s.startswith('0xfe00', end=-41) - - # endswith - assert s.endswith('0x00fe', start=-16) - assert not s.endswith('0x00fe', start=-15) - assert not s.endswith('0x00fe', end=-1) - assert s.endswith('0x00f', end=-4) - - # replace - s.replace('0xfe', '', end=-1) - assert s == '0x00121200fe' - s.replace('0x00', '', start=-24) - assert s == '0x001212fe' - - def test_rotate_start_and_end(self): - a = BitStream('0b110100001') - a.rol(1, 3, 6) - assert a == '0b110001001' - a.ror(1, start=-4) - assert a == '0b110001100' - a.rol(202, end=-5) - assert a == '0b001101100' - a.ror(3, end=4) - assert a == '0b011001100' - with pytest.raises(ValueError): - a.rol(5, start=-4, end=-6) - - def test_byte_swap_int(self): - s = pack('5*uintle:16', *range(10, 15)) - assert list(range(10, 15)) == s.unpack('5*uintle:16') - swaps = s.byteswap(2) - assert list(range(10, 15)) == s.unpack('5*uintbe:16') - assert swaps == 5 - s = BitStream('0xf234567f') - swaps = s.byteswap(1, start=4) - assert swaps == 3 - assert s == '0xf234567f' - s.byteswap(2, start=4) - assert s == '0xf452367f' - s.byteswap(2, start=4, end=-4) - assert s == '0xf234567f' - s.byteswap(3) - assert s == '0x5634f27f' - s.byteswap(2, repeat=False) - assert s == '0x3456f27f' - swaps = s.byteswap(5) - assert swaps == 0 - swaps = s.byteswap(4, repeat=False) - assert swaps == 1 - assert s == '0x7ff25634' - - def test_byte_swap_pack_code(self): - s = BitStream('0x0011223344556677') - swaps = s.byteswap('b') - assert s == '0x0011223344556677' - assert swaps == 8 - swaps = s.byteswap('>3h', repeat=False) - assert s == '0x1100332255446677' - assert swaps == 1 - - def test_byte_swap_iterable(self): - s = BitStream('0x0011223344556677') - swaps = s.byteswap(range(1, 4), repeat=False) - assert swaps == 1 - assert s == '0x0022115544336677' - swaps = s.byteswap([2], start=8) - assert s == '0x0011224455663377' - assert 3 == swaps - swaps = s.byteswap([2, 3], start=4) - assert swaps == 1 - assert s == '0x0120156452463377' - - def test_byte_swap_errors(self): - s = BitStream('0x0011223344556677') - with pytest.raises(ValueError): - s.byteswap('z') - with pytest.raises(ValueError): - s.byteswap(-1) - with pytest.raises(ValueError): - s.byteswap([-1]) - with pytest.raises(ValueError): - s.byteswap([1, 'e']) - with pytest.raises(ValueError): - s.byteswap('!h') - with pytest.raises(ValueError): - s.byteswap(2, start=-1000) - with pytest.raises(TypeError): - s.byteswap(5.4) - - def test_byte_swap_from_file(self): - s = BitStream(filename=os.path.join(THIS_DIR, 'smalltestfile')) - swaps = s.byteswap('2bh') - assert s == '0x0123674589abefcd' - assert swaps == 2 - - def test_bracket_expander(self): - be = bitstring.utils.expand_brackets - assert be('hello') == 'hello' - assert be('(hello)') == 'hello' - assert be('1*(hello)') == 'hello' - assert be('2*(hello)') == 'hello,hello' - assert be('1*(a,b)') == 'a,b' - assert be('2*(a,b)') == 'a,b,a,b' - assert be('2*(a),3*(b)') == 'a,a,b,b,b' - assert be('2*(a,b,3*(c,d),e)') == 'a,b,c,d,c,d,c,d,e,a,b,c,d,c,d,c,d,e' - with pytest.raises(ValueError): - _ = be('2*(x,y()') - - def test_bracket_tokens(self): - s = BitStream('3*(0x0, 0b1)') - assert s == '0x0, 0b1, 0x0, 0b1, 0x0, 0b1' - s = pack('2*(uint:12, 3*(uint:7, uint:6))', *range(3, 17)) - a = s.unpack('12, 7, 6, 7, 6, 7, 6, 12, 7, 6, 7, 6, 7, 6') - assert [x.uint for x in a] == list(range(3, 17)) - b = s.unpack('2*(12,3*(7,6))') - assert a == b - - def test_pack_code_dicts(self): - assert sorted(bitstring.utils.REPLACEMENTS_BE.keys()) == \ - sorted(bitstring.utils.REPLACEMENTS_LE.keys()) - assert sorted(bitstring.utils.REPLACEMENTS_BE.keys()) == \ - sorted(bitstring.utils.PACK_CODE_SIZE.keys()) - for key in bitstring.utils.PACK_CODE_SIZE: - be = pack(bitstring.utils.REPLACEMENTS_BE[key], 0) - le = pack(bitstring.utils.REPLACEMENTS_LE[key], 0) - assert be.len == bitstring.utils.PACK_CODE_SIZE[key] * 8 - assert le.len == be.len - - def test_unicode(self): - a = ConstBitStream(u'uint:12=34') - assert a.uint == 34 - a += u'0xfe' - assert a[12:] == '0xfe' - a = BitStream('0x1122') - c = a.byteswap(u'h') - assert c == 1 - assert a == u'0x2211' - - -class TestUnpackWithDict: - def test_length_keywords(self): - a = ConstBitStream('2*int:13=100, 0b111') - x, y, z = a.unpack('13, int:m, bin:q', m=13, q=3) - assert x == 'uint:13=100' - assert y == 100 - assert z == '111' - - def test_length_keywords_with_stretch(self): - a = ConstBitStream('0xff, 0b000, 0xf') - x, y, z = a.unpack('hex:a, bin, hex:b', a=8, b=4) - assert y == '000' - - def test_unused_keyword(self): - a = ConstBitStream('0b110') - x, = a.unpack('bin:3', notused=33) - assert x == '110' - - def test_length_keyword_errors(self): - a = pack('uint:p=33', p=12) - with pytest.raises(ValueError): - a.unpack('uint:p') - with pytest.raises(ValueError): - a.unpack('uint:p', p='a_string') - - -class TestReadWithDict: - def test_length_keywords(self): - s = BitStream('0x0102') - x, y = s.readlist('bits8, hex:b', b=4) - assert (x, y) == ('0x01', '0') - assert s.pos == 12 - - def test_bytes_keyword_problem(self): - s = BitStream('0x01') - x, = s.unpack('bytes:a', a=1) - assert x == b'\x01' - - s = BitStream('0x000ff00a') - x, y, z = s.unpack('12, bytes:x, bits', x=2) - assert (x.int, y, z) == (0, b'\xff\x00', '0xa') - - -class TestPeekWithDict: - def test_length_keywords(self): - s = BitStream('0x0102') - x, y = s.peeklist('8, hex:b', b=4) - assert (x, y) == ('0x01', '0') - assert s.pos == 0 - - -class TestBoolToken: - def test_interpretation(self): - a = ConstBitStream('0b1') - assert a.bool == True - assert a.read('bool') == True - assert a.unpack('bool')[0] == True - b = ConstBitStream('0b0') - assert b.bool == False - assert b.peek('bool') == False - assert b.unpack('bool')[0] == False - - def test_pack(self): - a = pack('bool=True') - b = pack('bool=False') - assert a.bool == True - assert b.bool == False - c = pack('4*bool', False, True, 'False', 'True') - assert c == '0b0101' - - def test_assignment(self): - a = BitStream() - a.bool = True - assert a.bool == True - a.hex = 'ee' - a.bool = False - assert a.bool == False - a.bool = 'False' - assert a.bool == False - a.bool = 'True' - assert a.bool == True - a.bool = 0 - assert a.bool == False - a.bool = 1 - assert a.bool == True - - def test_errors(self): - with pytest.raises(bitstring.CreationError): - pack('bool', 'hello') - with pytest.raises(bitstring.CreationError): - pack('bool=true') - with pytest.raises(bitstring.CreationError): - pack('True') - with pytest.raises(bitstring.CreationError): - pack('bool', 2) - with pytest.raises(bitstring.CreationError): - _ = pack('bool', 'hello') - with pytest.raises(bitstring.CreationError): - _ = pack('bool=true') - with pytest.raises(bitstring.CreationError): - _ = pack('True') - with pytest.raises(bitstring.CreationError): - _ = pack('bool', 2) - a = BitStream('0b11') - with pytest.raises(bitstring.InterpretError): - _ = a.bool - b = BitStream() - with pytest.raises(bitstring.InterpretError): - _ = b.bool - with pytest.raises(bitstring.CreationError): - b.bool = 'false' - - def test_length_with_bool_read(self): - a = ConstBitStream('0xf') - with pytest.raises(ValueError): - _ = a.read('bool:0') - with pytest.raises(ValueError): - _ = a.read('bool:2') - - -class TestReadWithIntegers: - def test_read_int(self): - a = ConstBitStream('0xffeedd') - b = a.read(8) - assert b.hex == 'ff' - assert a.pos == 8 - b = a.peek(8) - assert b.hex == 'ee' - assert a.pos == 8 - b = a.peek(1) - assert b == '0b1' - b = a.read(1) - assert b == '0b1' - - def test_read_int_list(self): - a = ConstBitStream('0xab, 0b110') - b, c = a.readlist([8, 3]) - assert b.hex == 'ab' - assert c.bin == '110' - - -# class FileReadingStrategy(unittest.TestCase): -# -# -# def testBitStreamIsAlwaysRead(self): -# filename = os.path.join(THIS_DIR, 'smalltestfile') -# a = BitStream(filename=filename) -# self.assertTrue(isinstance(a._datastore, bitstring.ByteStore)) -# with open(filename, 'rb') as f: -# b = BitStream(f) -# self.assertTrue(isinstance(b._datastore, bitstring.ByteStore)) -# -# -# def testBitsIsNeverRead(self): -# filename = os.path.join(THIS_DIR, 'smalltestfile') -# a = ConstBitStream(filename=filename) -# self.assertTrue(isinstance(a._datastore.rawarray, bitstring.MmapByteArray)) -# with open(filename, 'rb') as f: -# b = ConstBitStream(f) -# self.assertTrue(isinstance(b._datastore.rawarray, bitstring.MmapByteArray)) - - -class TestCount: - def test_count(self): - a = ConstBitStream('0xf0f') - assert a.count(True) == 8 - assert a.count(False) == 4 - - b = BitStream() - assert b.count(True) == 0 - assert b.count(False) == 0 - - def test_count_with_offset_data(self): - a = ConstBitStream('0xff0120ff') - b = a[1:-1] - assert b.count(1) == 16 - assert b.count(0) == 14 - - -class TestZeroBitReads: - def test_integer(self): - a = ConstBitStream('0x123456') - with pytest.raises(bitstring.InterpretError): - _ = a.read('uint:0') - with pytest.raises(bitstring.InterpretError): - _ = a.read('float:0') - - -class TestInitialiseFromBytes: - def test_bytes_behaviour(self): - a = ConstBitStream(b'uint:5=2') - b = ConstBitStream(b'') - c = ConstBitStream(bytes=b'uint:5=2') - assert a.bytes == b'uint:5=2' - assert not b - assert c == b'uint:5=2' - - def test_bytearray_behaviour(self): - a = ConstBitStream(bytearray(b'uint:5=2')) - b = ConstBitStream(bytearray(4)) - c = ConstBitStream(bytes=bytearray(b'uint:5=2')) - assert a.bytes == b'uint:5=2' - assert b == '0x00000000' - assert c.bytes == b'uint:5=2' - - -class TestCoverageCompletion: - def test_ue_read_error(self): - s = ConstBitStream('0b000000001') - with pytest.raises(bitstring.ReadError): - _ = s.read('ue') - - def test_overwrite_with_self(self): - s = BitStream('0b1101') - s.overwrite(s) - assert s == '0b1101' - - -class TestSubclassing: - - def test_is_instance(self): - b = BitStream() - assert isinstance(b, BitStream) - class SubBits(BitStream): - pass - a = SubBits() - assert isinstance(a, SubBits) - - def test_class_type(self): - class SubBits(BitStream): - pass - assert SubBits().__class__ == SubBits - - -class TestBytesProblems: - - def test_offset_but_no_length(self): - b = BitStream(bytes=b'\x00\xaa', offset=8) - assert b.hex == 'aa' - b = BitStream(bytes=b'\x00\xaa', offset=4) - assert b.hex == '0aa' - - def test_invert(self): - b = BitStream(bytes=b'\x00\xaa', offset=8, length=8) - assert b.hex == 'aa' - b.invert() - assert b.hex == '55' - - def test_prepend(self): - b = BitStream(bytes=b'\xaa\xbb', offset=8, length=4) - assert b.hex == 'b' - b.prepend('0xe') - assert b.hex == 'eb' - b = BitStream(bytes=b'\x00\xaa', offset=8, length=8) - b.prepend('0xee') - assert b.hex == 'eeaa' - - def test_byte_swap(self): - b = BitStream(bytes=b'\x01\x02\x03\x04', offset=8) - b.byteswap() - assert b == '0x040302' - - def test_bin_property(self): - b = BitStream(bytes=b'\x00\xaa', offset=8, length=4) - assert b.bin == '1010' - - -class TestLsb0Streaming: - - @classmethod - def setup_class(cls): - bitstring.lsb0 = True - - @classmethod - def teardown_class(cls): - bitstring.lsb0 = False - - def test_simple_bit_positions(self): - s = BitStream('0x00000f') - assert s.pos == 0 - v = s.read('uint:8') - assert v == 15 - assert s.pos == 8 - v = s.read(10) - assert v == Bits(10) - assert s.pos == 18 - - def test_bit_pos_after_find(self): - s = BitStream('0b01100001000011 0000') - s.find('0b11', start=1) - assert s.pos == 4 - - def test_iter(self): - s = BitStream('0b11000') - assert list(s) == [False, False, False, True, True] - - def test_bit_pos_after_rfind(self): - s = BitStream('0b011 000010000110000') - s.rfind('0b11') - assert s.pos == 15 - - def test_bit_pos_after_findall(self): - pass - - def test_bit_pos_after_insert(self): - pass - - def test_bit_pos_after_overwrite(self): - pass - - def test_bit_pos_after_replace(self): - pass - - def test_read_list(self): - a = BitStream('0x0123456789abcdef') - - vals = a.readlist('uint:4, uint:4, uint:24, uint:12, uint:12, uint:8') - assert vals == [15, 14, 0x89abcd, 0x567, 0x234, 1] - - -class TestLsb0PackingUnpacking: - - @classmethod - def setup_class(cls): - bitstring.lsb0 = True - - @classmethod - def teardown_class(cls): - bitstring.lsb0 = False - - def test_bin(self): - lsb0 = bitstring.pack('2*b4', '0b0000', '1111') - assert lsb0 == '0b11110000' - a, b = lsb0.unpack('2*h4') - assert [a, b] == ['0', 'f'] - a, b = lsb0.unpack('2*bits4') - assert [a, b] == ['0x0', '0xf'] - a, b = lsb0.unpack('2*bin4') - assert [a, b] == ['0000', '1111'] - - def test_float(self): - lsb0 = bitstring.pack('2*bfloat', 0.5, 15) - assert lsb0 == '0x4170 3f00' - a, b = lsb0.unpack('2*bfloat') - assert [a, b] == [0.5, 15] - - def test_simplest(self): - lsb0 = bitstring.pack('uint:2', 1) - assert lsb0.unpack('uint:2') == [1] - lsb0 = bitstring.pack('0xab, 0xc') - assert lsb0.unpack('hex8, hex4') == ['ab', 'c'] - - def test_slightly_harder(self): - lsb0 = bitstring.pack('float:32, hex', 0.25, 'ac') - x = lsb0.unpack('float:32, hex') - assert x == [0.25, 'ac'] - - def test_more_complex(self): - lsb0 = bitstring.pack('uint:10, hex, int:13, 0b11', 130, '3d', -23) - x = lsb0.unpack('uint:10, hex, int:13, bin:2') - assert x == [130, '3d', -23, '11'] - - def test_golomb_codes(self): - v = [10, 8, 6, 4, 100, -9] - # Exp-Golomb codes can only be read in msb0 mode. So also doesn't - # make sense for creation with pack - with pytest.raises(bitstring.CreationError): - _ = bitstring.pack('5*ue, sie', *v) - # with self.assertRaises(bitstring.CreationError): - # _ = BitStream('ue=34') - lsb0 = BitStream('0b0010010') - with pytest.raises(bitstring.ReadError): - _ = lsb0.unpack('5*ue, sie') - with pytest.raises(bitstring.ReadError): - _ = lsb0.read('ue') - with pytest.raises(bitstring.ReadError): - _ = lsb0.read('uie') - with pytest.raises(bitstring.ReadError): - _ = lsb0.read('se') - with pytest.raises(bitstring.ReadError): - _ = lsb0.read('sie') - - -class TestRepr: - - def test_without_pos(self): - a = BitStream('0x12345', pos=0) - assert repr(a) == "BitStream('0x12345')" - - def test_with_pos(self): - a = BitStream('0b00111', pos=-1) - assert a.pos == 4 - assert repr(a) == "BitStream('0b00111', pos=4)" - - -class TestFormat: - - def test_simple_format_strings(self): - a = Bits('0xabc') - s = f'{a}' - assert s == '0xabc' - a += '0b0' - assert f'{a}' == '0b1010101111000' - b = BitStream(10, pos=4) - assert f'{b}' == '0b0000000000' - c = BitStream(filename=os.path.join(THIS_DIR, 'test.m1v')) - assert f'{c}'[0:10] == '0x000001b3' - - def test_format_strings_with_interpretation(self): - a = Bits('0xf') - assert f'{a.bin}' == '1111' - - -class TestCacheingIssues: - - def test_cache_with_offset(self): - y = BitStream('0xdeadbeef1000') - with pytest.raises(bitstring.CreationError): - x = BitStream('0xdeadbeef1000', offset=8) - - def test_cache_with_pos(self): - y = BitStream('0xdeadbeef1001', pos=3) - assert y.pos == 3 - x = BitStream('0xdeadbeef1001', pos=5) - assert x.pos == 5 - - def test_cache_with_length(self): - y = BitStream('0xdeadbeef002') - with pytest.raises(bitstring.CreationError): - x = BitStream('0xdeadbeef002', length=16) - - -def test_unpack_error(): - format_with_commas = ',bytes:2,,bytes:1,' - dp = BitStream(hex='010203').unpack(fmt=format_with_commas) - assert dp == [b'\x01\x02', b'\x03'] - - -def test_add_pos_issue(): - x = BitStream() - y = x + Bits('0xff') - assert x.pos == 0 - assert y == '0xff' - z = x + bitstring.BitArray('0xff') - assert z == '0xff' - q = x + ConstBitStream('0xff') - assert q == '0xff' - - xx = ConstBitStream() - yy = xx + Bits('0xff') - zz = xx + bitstring.BitArray('0xff') - qq = xx + BitStream('0xff') - assert yy == zz == qq == '0xff' - diff --git a/tests/test_bitstring.py b/tests/test_bitstring.py index 98babebb..91ee4649 100644 --- a/tests/test_bitstring.py +++ b/tests/test_bitstring.py @@ -2,15 +2,11 @@ """ Module-level unit tests. """ -import io -from unittest import mock -from contextlib import redirect_stdout import bitstring import copy from collections import abc import sys import os -from bitstring import __main__ sys.path.insert(0, '..') @@ -20,15 +16,15 @@ class TestModuleData: def test_all(self): - exported = ['ConstBitStream', 'BitStream', 'BitArray', + exported = ['Reader', 'BitArray', 'Bits', 'pack', 'Error', 'ReadError', 'Array', - 'InterpretError', 'ByteAlignError', 'CreationError', 'bytealigned', 'lsb0', 'Dtype', 'options'] + 'InterpretError', 'ByteAlignError', 'CreationError', 'Dtype', 'options'] assert set(bitstring.__all__) == set(exported) def test_pyproject_version(self): filename = os.path.join(THIS_DIR, '../pyproject.toml') try: - with open(filename, 'r') as pyprojectfile: + with open(filename) as pyprojectfile: found = False for line in pyprojectfile.readlines(): if line.startswith("version"): @@ -39,35 +35,38 @@ def test_pyproject_version(self): except FileNotFoundError: pass # Doesn't run on CI. + def test_module_version_matches_pyproject_exactly(self): + filename = os.path.join(THIS_DIR, '../pyproject.toml') + try: + with open(filename) as pyprojectfile: + for line in pyprojectfile: + if line.startswith("version"): + pyproject_version = line.split("=", 1)[1].strip().strip('"') + assert bitstring.__version__ == pyproject_version + return + except FileNotFoundError: + pass # Doesn't run on CI. + class TestCopy: def test_const_bit_array_copy(self): - cba = bitstring.Bits(100) + cba = bitstring.Bits.from_zeros(100) cba_copy = copy.copy(cba) assert cba is cba_copy def test_bit_array_copy(self): - ba = bitstring.BitArray(100) + ba = bitstring.BitArray.from_zeros(100) ba_copy = copy.copy(ba) assert not ba is ba_copy assert not ba._bitstore is ba_copy._bitstore assert ba == ba_copy - def test_const_bit_stream_copy(self): - cbs = bitstring.ConstBitStream(100) - cbs.pos = 50 - cbs_copy = copy.copy(cbs) - assert cbs_copy.pos == 0 - assert cbs._bitstore is cbs_copy._bitstore - assert cbs == cbs_copy - - def test_bit_stream_copy(self): - bs = bitstring.BitStream(100) - bs.pos = 50 - bs_copy = copy.copy(bs) - assert bs_copy.pos == 0 - assert not bs._bitstore is bs_copy._bitstore - assert bs == bs_copy + def test_reader_copy(self): + bits = bitstring.Bits.from_zeros(100) + r = bitstring.Reader(bits, 50) + r_copy = copy.copy(r) + assert r_copy.pos == 50 + assert r_copy.bits is bits class TestInterning: @@ -78,72 +77,13 @@ def test_bits(self): c = bitstring.Bits('0b1111') assert not a is c - def test_cbs(self): - a = bitstring.ConstBitStream('0b11000') - b = bitstring.ConstBitStream('0b11000') + def test_bits_cache(self): + a = bitstring.Bits('0b11000') + b = bitstring.Bits('0b11000') assert a._bitstore is b._bitstore assert not a is b -class TestLSB0: - def test_getting_and_setting(self): - assert bitstring.lsb0 == False - bitstring.lsb0 = True - assert bitstring.lsb0 == True - bitstring.lsb0 = False - assert bitstring.lsb0 == False - - -class TestMain: - def test_running_module_directly_help(self): - with redirect_stdout(io.StringIO()) as f: - with mock.patch('sys.argv', ['bitstring.py', '-h']): - bitstring.__main__.main() - s = f.getvalue() - assert s.find("command-line parameters") >= 0 - - with redirect_stdout(io.StringIO()) as f: - with mock.patch('sys.argv', ['renamed.py']): - bitstring.__main__.main() - s = f.getvalue() - assert s.find("command-line parameters") >= 0 - - def test_running_module_with_single_parameter(self): - with redirect_stdout(io.StringIO()) as f: - with mock.patch('sys.argv', ['', 'uint:12=352']): - bitstring.__main__.main() - s = f.getvalue() - assert s == '0x160\n' - - def test_running_module_with_single_parameter_and_interpretation(self): - with redirect_stdout(io.StringIO()) as f: - with mock.patch('sys.argv', ['ignored', 'u12=352', 'i']): - bitstring.__main__.main() - s = f.getvalue() - assert s == '352\n' - - def test_running_module_with_multiple_parameters(self): - with redirect_stdout(io.StringIO()) as f: - with mock.patch('sys.argv', ['b.py', 'uint12=352', '0b101', '0o321', 'f32=51', 'bool=1']): - bitstring.__main__.main() - s = f.getvalue() - assert s == '0x160ad1424c0000, 0b1\n' - - def test_running_module_with_multiple_parameters_and_interpretation(self): - with redirect_stdout(io.StringIO()) as f: - with mock.patch('sys.argv', ['b.py', 'ue=1000', '0xff.bin']): - bitstring.__main__.main() - s = f.getvalue() - assert s == '000000000111110100111111111\n' - - def test_short_interpretations(self): - with redirect_stdout(io.StringIO()) as f: - with mock.patch('sys.argv', ['b.py', 'bin=001.b']): - bitstring.__main__.main() - s = f.getvalue() - assert s == '001\n' - - class TestABCs: def test_base_classes(self): # The classes deliberately do not conform to the sequence ABCs. @@ -156,13 +96,9 @@ def test_base_classes(self): assert not isinstance(bitarray, abc.MutableSequence) assert not isinstance(bitarray, abc.Sequence) - constbitstream = bitstring.ConstBitStream() - assert not isinstance(constbitstream, abc.Sequence) - assert not isinstance(constbitstream, abc.MutableSequence) - - bitstream = bitstring.BitArray() - assert not isinstance(bitstream, abc.MutableSequence) - assert not isinstance(bitstream, abc.Sequence) + bitarray = bitstring.BitArray() + assert not isinstance(bitarray, abc.MutableSequence) + assert not isinstance(bitarray, abc.Sequence) class TestNoFixedLengthPackingBug: @@ -180,29 +116,25 @@ def test_packing_hex_with_no_length(self): assert a.hex == 'abcd' def test_reading_bytes_with_no_length(self): - a = bitstring.BitStream(b'hello') + a = bitstring.Reader(bitstring.Bits(b'hello')) b = a.read('bytes') assert b == b'hello' def test_reading_bin_with_no_length(self): - a = bitstring.BitStream('0b1101') + a = bitstring.Reader(bitstring.Bits('0b1101')) b = a.read('bin') assert b == '1101' - def test_reading_uint_with_no_length(self): - a = bitstring.BitStream('0b1101') - b = a.read('uint') + def test_reading_u_with_no_length(self): + a = bitstring.Reader(bitstring.Bits('0b1101')) + b = a.read('u') assert b == 13 - def test_reading_float_with_no_length(self): - a = bitstring.BitStream(float=14, length=16) - b = a.read('float') + def test_reading_f_with_no_length(self): + a = bitstring.Reader(bitstring.Bits(f=14, length=16)) + b = a.read('f') assert b == 14.0 -def test_rust_core(): - using_rust = bitstring.options.using_rust_core - x = bitstring.BitStream('0x1') - if hasattr(x._bitstore, "_bitarray"): - assert using_rust is False - else: - assert using_rust is True \ No newline at end of file + def test_pack_returns_bits(self): + a = bitstring.pack('u8=1') + assert type(a) is bitstring.Bits diff --git a/tests/test_constbitstream.py b/tests/test_constbitstream.py deleted file mode 100644 index ef15cca8..00000000 --- a/tests/test_constbitstream.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/env python - -import pytest -import sys -import bitstring -import io -import os -from bitstring import ConstBitStream as CBS -import platform - -sys.path.insert(0, '..') - - -THIS_DIR = os.path.dirname(os.path.abspath(__file__)) - -class TestAll: - def test_from_file(self): - s = CBS(filename=os.path.join(THIS_DIR, 'test.m1v')) - assert s[0:32].hex == '000001b3' - assert s.read(8 * 4).hex == '000001b3' - width = s.read(12).uint - height = s.read(12).uint - assert (width, height) == (352, 288) - - def test_from_file_with_offset_and_length(self): - s = CBS(filename=os.path.join(THIS_DIR, 'test.m1v'), offset=24, length=8) - assert s.h == 'b3' - reconstructed = '' - for bit in s: - reconstructed += '1' if bit is True else '0' - assert reconstructed == s.bin - - -class TestInterleavedExpGolomb: - def test_reading(self): - s = CBS(uie=333) - a = s.read('uie') - assert a == 333 - s = CBS('uie=12, sie=-9, sie=9, uie=1000000') - u = s.unpack('uie, 2*sie, uie') - assert u == [12, -9, 9, 1000000] - - def test_reading_errors(self): - s = CBS(10) - with pytest.raises(bitstring.ReadError): - s.read('uie') - assert s.pos == 0 - with pytest.raises(bitstring.ReadError): - s.read('sie') - assert s.pos == 0 - - -class TestReadTo: - def test_byte_aligned(self): - a = CBS('0xaabb00aa00bb') - b = a.readto('0x00', bytealigned=True) - assert b == '0xaabb00' - assert a.bytepos == 3 - b = a.readto('0xaa', bytealigned=True) - assert b == '0xaa' - with pytest.raises(bitstring.ReadError): - b.readto('0xcc', bytealigned=True) - - def test_not_aligned(self): - a = CBS('0b00111001001010011011') - a.pos = 1 - assert a.readto('0b00') == '0b011100' - assert a.readto('0b110') == '0b10010100110' - with pytest.raises(ValueError): - a.readto('') - - def test_disallow_integers(self): - a = CBS('0x0f') - with pytest.raises(ValueError): - a.readto(4) - - def test_reading_lines(self): - s = b"This is a test\nof reading lines\nof text\n" - b = CBS(bytes=s) - n = bitstring.Bits(bytes=b'\n') - assert b.readto(n).bytes == b'This is a test\n' - assert b.readto(n).bytes == b'of reading lines\n' - assert b.readto(n).bytes == b'of text\n' - - -class TestSubclassing: - - def test_is_instance(self): - class SubBits(CBS): - pass - a = SubBits() - assert isinstance(a, SubBits) - - def test_class_type(self): - class SubBits(CBS): - pass - assert SubBits().__class__ == SubBits - - -class TestPadToken: - - def test_read(self): - s = CBS('0b100011110001') - a = s.read('pad:1') - assert a == None - assert s.pos == 1 - a = s.read(3) - assert a == CBS('0b000') - a = s.read('pad:1') - assert a == None - assert s.pos == 5 - - def test_read_list(self): - s = CBS.fromstring('0b10001111001') - t = s.readlist('pad:1, uint:3, pad:4, uint:3') - assert t == [0, 1] - s.pos = 0 - t = s.readlist('pad:1, pad:5') - assert t == [] - assert s.pos == 6 - s.pos = 0 - t = s.readlist('pad:1, bin, pad:4, uint:3') - assert t == ['000', 1] - s.pos = 0 - t = s.readlist('pad, bin:3, pad:4, uint:3') - assert t == ['000', 1] - - -class TestReadingBytes: - - def test_unpacking_bytes(self): - s = CBS(80) - t = s.unpack('bytes:1') - assert t[0] == b'\x00' - a, b, c = s.unpack('bytes:1, bytes, bytes2') - assert a == b'\x00' - assert b == b'\x00'*7 - assert c == b'\x00'*2 - - def test_unpacking_bytes_with_keywords(self): - s = CBS('0x55'*10) - t = s.unpack('pad:a, bytes:b, bytes, pad:a', a=4, b=6) - assert t == [b'\x55'*6, b'\x55'*3] - - -class TestReadingBitsAsDefault: - - def test_read_bits(self): - s = CBS('uint:31=14') - v = s.read(31) - assert v.uint == 14 - s.pos = 0 - - def test_read_list_bits(self): - s = CBS('uint:5=3, uint:3=0, uint:11=999') - v = s.readlist([5, 3, 11]) - assert [x.uint for x in v] == [3, 0, 999] - s.pos = 0 - v = s.readlist(['5', '3', 11]) - assert [x.uint for x in v] == [3, 0, 999] - - -class TestLsb0Reading: - - @classmethod - def setup_class(cls): - bitstring.lsb0 = True - - @classmethod - def teardown_class(cls): - bitstring.lsb0 = False - - def test_reading_hex(self): - s = CBS('0xabcdef') - assert s.read('hex:4') == 'f' - assert s.read(4) == '0xe' - assert s.pos == 8 - - def test_reading_oct(self): - s = CBS('0o123456') - assert s.read('o6') == '56' - assert s.pos == 6 - - def test_reading_bin(self): - s = CBS('0b00011') - assert s.read('bin:3') == '011' - assert s.pos == 3 - - def test_reading_bytes(self): - s = CBS(bytes=b'54321') - assert s.pos == 0 - s.pos = 8 - assert s.read('bytes:2') == b'32' - - -class TestBytesIOCreation: - - def test_simple_creation(self): - f = io.BytesIO(b"\x12\xff\x77helloworld") - s = CBS(f) - assert s[0:8] == '0x12' - assert len(s) == 13 * 8 - s = CBS(f, offset=8, length=12) - assert s == '0xff7' - - def test_exceptions(self): - f = io.BytesIO(b"123456789") - _ = CBS(f, length=9*8) - with pytest.raises(bitstring.CreationError): - _ = CBS(f, length=9*8 + 1) - with pytest.raises(bitstring.CreationError): - _ = CBS(f, length=9*8, offset=1) - - -class TestCreationWithPos: - - def test_default_creation(self): - s = CBS('0xabc') - assert s.pos == 0 - - def test_positive_pos(self): - s = CBS('0xabc', pos=0) - assert s.pos == 0 - s = CBS('0xabc', pos=1) - assert s.pos == 1 - s = CBS('0xabc', pos=12) - assert s.pos == 12 - with pytest.raises(bitstring.CreationError): - _ = CBS('0xabc', pos=13) - - def test_negative_pos(self): - s = CBS('0xabc', pos=-1) - assert s.pos == 11 - s = CBS('0xabc', pos=-12) - assert s.pos == 0 - with pytest.raises(bitstring.CreationError): - _ = CBS('0xabc', pos=-13) - - def test_string_representation(self): - s = CBS('0b110', pos=2) - assert s.__repr__() == "ConstBitStream('0b110', pos=2)" - - def test_string_representation_from_file(self): - filename = os.path.join(THIS_DIR, 'test.m1v') - s = CBS(filename=filename, pos=2001) - assert s.__repr__() == f"ConstBitStream(filename={repr(str(filename))}, length=1002400, pos=2001)" - s.pos = 0 - assert s.__repr__() == f"ConstBitStream(filename={repr(str(filename))}, length=1002400)" - - -def test_windows_file_lock_bug(): - path = os.path.join(THIS_DIR, 'temp_unit_test_file') - # Create the file - with open(path, mode='w') as f: - f.write('Hello') - # Will this lock it? - _ = CBS(filename=path) - - try: - with open(path, mode='w') as _: - pass - except OSError: - if platform.system() == 'Windows': - # Expected failure. See bug #308 - pass - -def test_readerrors(): - s = CBS('0b110') - s.read(3) - with pytest.raises(bitstring.ReadError): - _ = s.read(1) - s.pos = 1 - with pytest.raises(bitstring.ReadError): - _ = s.read('u3') \ No newline at end of file diff --git a/tests/test_dtypes.py b/tests/test_dtypes.py index 97767c69..f3bf4aa6 100644 --- a/tests/test_dtypes.py +++ b/tests/test_dtypes.py @@ -1,8 +1,7 @@ - import pytest import sys import bitstring as bs -from bitstring import Dtype, DtypeDefinition, Bits +from bitstring import Dtype, DtypeDefinition sys.path.insert(0, '..') @@ -21,7 +20,7 @@ def test_setting_bool(self): def test_reading(self): b = Dtype('u8') - a = bs.BitStream('0xff00ff') + a = bs.Reader(bs.Bits('0xff00ff')) x = a.read(b) assert x == 255 x = a.read(b) @@ -29,26 +28,73 @@ def test_reading(self): def test_setting_with_length(self): d = Dtype('uint', 12) - assert str(d) == 'uint12' + assert str(d) == 'u12' assert d.length == 12 - assert d.name == 'uint' - - def test_build_errors(self): + assert d.name == 'u' + + def test_short_numeric_names_are_canonical(self): + assert Dtype('uint12') == Dtype('u12') + assert Dtype('int12') == Dtype('i12') + assert Dtype('float16') == Dtype('f16') + assert str(Dtype('uint12')) == 'u12' + assert str(Dtype('int12')) == 'i12' + assert str(Dtype('float16')) == 'f16' + assert repr(Dtype('uint', 12)) == "Dtype('u', 12)" + assert repr(Dtype('int', 12)) == "Dtype('i', 12)" + assert repr(Dtype('float', 16)) == "Dtype('f', 16)" + + def test_short_endian_numeric_names_are_canonical(self): + assert Dtype('uintbe16') == Dtype('ube16') + assert Dtype('uintle16') == Dtype('ule16') + assert Dtype('uintne16') == Dtype('une16') + assert Dtype('intbe16') == Dtype('ibe16') + assert Dtype('intle16') == Dtype('ile16') + assert Dtype('intne16') == Dtype('ine16') + assert Dtype('floatbe16') == Dtype('fbe16') == Dtype('f16') + assert Dtype('floatle16') == Dtype('fle16') + assert Dtype('floatne16') == Dtype('fne16') + assert str(Dtype('uintbe16')) == 'ube16' + assert str(Dtype('uintle16')) == 'ule16' + assert str(Dtype('uintne16')) == 'une16' + assert str(Dtype('intbe16')) == 'ibe16' + assert str(Dtype('intle16')) == 'ile16' + assert str(Dtype('intne16')) == 'ine16' + assert str(Dtype('floatbe16')) == 'f16' + assert str(Dtype('fbe16')) == 'f16' + assert str(Dtype('floatle16')) == 'fle16' + assert str(Dtype('floatne16')) == 'fne16' + + def test_short_endian_keyword_names_are_canonical(self): + assert bs.Bits(ube=1, length=16) == bs.Bits(uintbe=1, length=16) + assert bs.Bits(ule=1, length=16) == bs.Bits(uintle=1, length=16) + assert bs.Bits(une=1, length=16) == bs.Bits(uintne=1, length=16) + assert bs.Bits(ibe=-1, length=16) == bs.Bits(intbe=-1, length=16) + assert bs.Bits(ile=-1, length=16) == bs.Bits(intle=-1, length=16) + assert bs.Bits(ine=-1, length=16) == bs.Bits(intne=-1, length=16) + assert bs.Bits(fbe=1.5, length=32) == bs.Bits(floatbe=1.5, length=32) + assert bs.Bits(fle=1.5, length=32) == bs.Bits(floatle=1.5, length=32) + assert bs.Bits(fne=1.5, length=32) == bs.Bits(floatne=1.5, length=32) + + def test_pack_errors(self): dtype = Dtype('uint8') value = 'not_an_integer' with pytest.raises(ValueError): - dtype.build(value) + dtype.pack(value) - def test_build(self): + def test_pack(self): dtype = Dtype('se') - x = dtype.build(10001) + x = dtype.pack(10001) assert x.se == 10001 - def test_parse(self): + def test_unpack(self): dtype = Dtype('uint:12') - x = dtype.parse('0x3ff') + x = dtype.unpack('0x3ff') assert x == 1023 + def test_old_pack_unpack_names_removed(self): + assert not hasattr(Dtype, 'build') + assert not hasattr(Dtype, 'parse') + def test_immutability(self): d = Dtype('e3m2mxfp') with pytest.raises(AttributeError): @@ -60,7 +106,7 @@ def test_immutability(self): def test_variable_lengths(self): d = Dtype('ue') - a = bs.BitStream().join([d.build(v) for v in [1, 100, 3, 17, 4]]) + a = bs.Reader(bs.Bits().join([d.pack(v) for v in [1, 100, 3, 17, 4]])) assert a.read(d) == 1 assert a.read(d) == 100 assert a.read(d) == 3 @@ -74,26 +120,26 @@ def test_variable_lengths(self): assert a.read(ds) == -51 assert a.read(ds) == -12 - def test_building_bits(self): + def test_packing_bits(self): d = Dtype('bits3') - a = d.build('0b101') + a = d.pack('0b101') assert a == '0b101' with pytest.raises(ValueError): - d.build('0b1010') + d.pack('0b1010') - def test_building_bin(self): + def test_packing_bin(self): d = Dtype('bin9') - a = d.build('0b000111000') + a = d.pack('0b000111000') assert a == '0b000111000' with pytest.raises(ValueError): - d.build('0b0001110000') + d.pack('0b0001110000') - def test_building_ints(self): + def test_packing_ints(self): d = Dtype('i3') - a = d.build(-3) + a = d.pack(-3) assert a == '0b101' with pytest.raises(ValueError): - d.build(4) + d.pack(4) class TestChangingTheRegister: @@ -108,20 +154,23 @@ def test_retrieving_meta_dtype(self): def test_removing_type(self): r = bs.dtype_register + bfloat = r['bfloat'] del r['bfloat'] with pytest.raises(KeyError): i = r['bfloat'] with pytest.raises(KeyError): del r['penguin'] + r.add_dtype(bfloat) class TestCreatingNewDtypes: def test_new_alias(self): bs.dtype_register.add_dtype_alias('bin', 'cat') - a = bs.BitStream('0b110110') + a = bs.BitArray('0b110110') + r = bs.Reader(a) assert a.cat == '110110' - assert a.read('cat4') == '1101' + assert r.read('cat4') == '1101' a.cat = '11110000' assert a.unpack('cat') == ['11110000'] @@ -140,9 +189,10 @@ def get_fn(bs): return bs.count(1) md = DtypeDefinition('counter', None, get_fn) bs.dtype_register.add_dtype(md) - a = bs.BitStream('0x010f') + a = bs.BitArray('0x010f') + r = bs.Reader(a) assert a.counter == 5 - assert a.readlist('2*counter8') == [1, 4] + assert r.readlist('2*counter8') == [1, 4] assert a.unpack('counter7, counter') == [0, 5] with pytest.raises(AttributeError): a.counter = 4 diff --git a/tests/test_fp8.py b/tests/test_fp8.py index 107c9f27..cef7dce1 100644 --- a/tests/test_fp8.py +++ b/tests/test_fp8.py @@ -5,7 +5,7 @@ import array import math import bitstring -from bitstring import Bits, BitArray, BitStream, Dtype +from bitstring import Bits, BitArray, Dtype, Reader from bitstring.fp8 import p4binary_fmt, p3binary_fmt from bitstring.mxfp import e4m3mxfp_saturate_fmt, e5m2mxfp_saturate_fmt, e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt from gfloat.formats import (format_info_ocp_e4m3, format_info_ocp_e5m2, format_info_p3109, format_info_ocp_e3m2, @@ -44,7 +44,7 @@ def test_reassignment(self): assert a.p3binary == 0.0 def test_reading(self): - a = BitStream('0x00fff') + a = Reader(Bits('0x00fff')) x = a.read('p3binary') assert x == 0.0 assert a.pos == 8 @@ -55,7 +55,7 @@ def test_reading(self): def test_read_list(self): v = [-6, -2, 0.125, 7, 10] a = bitstring.pack('5*p4binary', *v) - vp = a.readlist('5*p4binary') + vp = Reader(a).readlist('5*p4binary') assert v == vp def test_interpretations(self): @@ -73,7 +73,7 @@ def createLUT_for_int8_to_float(exp_bits, bias) -> array.array[float]: """Create a LUT to convert an int in range 0-255 representing a float8 into a Python float""" i2f = [] for i in range(256): - b = BitArray(uint=i, length=8) + b = BitArray(u=i, length=8) sign = b[0] exponent = b[1:1 + exp_bits].u significand = b[1 + exp_bits:] @@ -284,13 +284,13 @@ def test_round_to_nearest(self): def test_compare_mxint8_with_gfloat(): for i in range(1 << 8): - f = Dtype('mxint8').parse(BitArray(uint=i, length=8)) + f = Dtype('mxint8').unpack(BitArray(u=i, length=8)) g = gfloat.decode_float(format_info_ocp_int8, i).fval assert f == g def test_compare_e8m0_with_gfloat(): for i in range(1 << 8): - f = Dtype('e8m0mxfp').parse(BitArray(uint=i, length=8)) + f = Dtype('e8m0mxfp').unpack(BitArray(u=i, length=8)) g = gfloat.decode_float(format_info_ocp_e8m0, i).fval if math.isnan(g): assert math.isnan(f) @@ -325,11 +325,10 @@ def test_rounding_consistent_to_gfloat(): for fi, dt in [[format_info_p3109(4), Dtype('p4binary')], [format_info_p3109(3), Dtype('p3binary')]]: for i in range(0, 1 << 16): - f = BitArray(uint=i, length=16).float - mine = dt.parse(dt.build(f)) + f = BitArray(u=i, length=16).float + mine = dt.unpack(dt.pack(f)) theirs = gfloat.round_float(fi, f) if math.isnan(mine): assert math.isnan(theirs) else: assert mine == theirs - diff --git a/tests/test_generated.py b/tests/test_generated.py new file mode 100644 index 00000000..ae3ed29c --- /dev/null +++ b/tests/test_generated.py @@ -0,0 +1,228 @@ +# LLM generated test cases +import bitstring + + +ConstBitStore = bitstring.bitstore.ConstBitStore +MutableBitStore = bitstring.bitstore.MutableBitStore + + +def test_mutable_bitstore_ilshift_keeps_binding_and_returns_self() -> None: + bs = MutableBitStore.from_bin("1011") + original_id = id(bs) + + bs <<= 1 + + assert id(bs) == original_id + assert bs.to_bin() == "0110" + + +def test_mutable_bitstore_irshift_keeps_binding_and_returns_self() -> None: + bs = MutableBitStore.from_bin("1011") + original_id = id(bs) + + bs >>= 1 + + assert id(bs) == original_id + assert bs.to_bin() == "0101" + + +def test_const_bitstore_eq_non_bitstore_returns_false() -> None: + bs = ConstBitStore.from_bin("1") + assert (bs == 1) is False + + +def test_mutable_bitstore_eq_non_bitstore_returns_false() -> None: + bs = MutableBitStore.from_bin("1") + assert (bs == 1) is False + + +def test_dtype_scaled_instances_compare_distinct() -> None: + from bitstring.dtypes import Dtype + + a = Dtype("uint8", scale=2) + b = Dtype("uint8", scale=3) + + assert a != b + assert len({a, b}) == 2 + + +def test_dtype_ube_zero_length_rejected() -> None: + from bitstring.dtypes import Dtype + import pytest + + with pytest.raises(ValueError): + _ = Dtype("ube0") + + +def test_array_fromfile_reads_from_current_file_position(tmp_path) -> None: + from bitstring.array_ import Array + + p = tmp_path / "fromfile.bin" + p.write_bytes(bytes([1, 2, 3, 4])) + + with p.open("rb") as f: + f.seek(1) + a = Array("uint8") + a.fromfile(f, 2) + + assert a.tolist() == [2, 3] + + +def test_pack_with_zero_repeated_group_needs_no_values() -> None: + s = bitstring.pack("0*(uint:8)") + assert len(s) == 0 + + +def test_array_iadd_with_array_is_in_place() -> None: + from bitstring.array_ import Array + + a = Array("int8", [1, 2]) + alias = a + a += Array("int8", [3, 4]) + + assert a is alias + assert alias.tolist() == [4, 6] + + +def test_array_fromfile_negative_n_rejected(tmp_path) -> None: + from bitstring.array_ import Array + import pytest + + p = tmp_path / "fromfile_negative.bin" + p.write_bytes(bytes([1, 2, 3])) + + with p.open("rb") as f: + a = Array("uint8") + with pytest.raises(ValueError): + a.fromfile(f, -1) + + +def test_dtype_ube_negative_length_rejected() -> None: + from bitstring.dtypes import Dtype + import pytest + + with pytest.raises(ValueError): + _ = Dtype("ube", -8) + + +def test_pack_negative_repeat_factor_rejected() -> None: + import pytest + + with pytest.raises(ValueError): + _ = bitstring.pack("-2*uint:8") + + +def test_options_constructor_does_not_reset_singleton_state() -> None: + from bitstring.bitstring_options import Options + + old_bytealigned = bitstring.options.bytealigned + try: + bitstring.options.bytealigned = True + Options() + assert bitstring.options.bytealigned is True + finally: + bitstring.options.bytealigned = old_bytealigned + + +def test_mxfp_overflow_option_removed() -> None: + assert not hasattr(bitstring.options, "mxfp_overflow") + + +def test_array_insert_very_negative_index_clamps_to_start() -> None: + from bitstring.array_ import Array + + a = Array("uint8", [10, 20, 30]) + a.insert(-100, 5) + assert a.tolist() == [5, 10, 20, 30] + + +def test_array_isub_with_array_is_in_place() -> None: + from bitstring.array_ import Array + + a = Array("int8", [5, 7]) + alias = a + a -= Array("int8", [2, 3]) + + assert a is alias + assert alias.tolist() == [3, 4] + + +def test_array_imul_with_array_is_in_place() -> None: + from bitstring.array_ import Array + + a = Array("int8", [2, 3]) + alias = a + a *= Array("int8", [4, 5]) + + assert a is alias + assert alias.tolist() == [8, 15] + + +def test_array_ifloordiv_with_array_is_in_place() -> None: + from bitstring.array_ import Array + + a = Array("int8", [8, 15]) + alias = a + a //= Array("int8", [2, 5]) + + assert a is alias + assert alias.tolist() == [4, 3] + + +def test_array_itruediv_with_array_is_in_place() -> None: + from bitstring.array_ import Array + + a = Array("float16", [8.0, 15.0]) + alias = a + a /= Array("float16", [2.0, 5.0]) + + assert a is alias + assert alias.tolist() == [4.0, 3.0] + + +def test_array_ilshift_with_array_is_in_place() -> None: + from bitstring.array_ import Array + + a = Array("uint8", [1, 2]) + alias = a + a <<= Array("uint8", [3, 2]) + + assert a is alias + assert alias.tolist() == [8, 8] + + +def test_array_irshift_with_array_is_in_place() -> None: + from bitstring.array_ import Array + + a = Array("uint8", [16, 18]) + alias = a + a >>= Array("uint8", [3, 1]) + + assert a is alias + assert alias.tolist() == [2, 9] + + +def test_array_imod_with_array_is_in_place() -> None: + from bitstring.array_ import Array + + a = Array("uint8", [20, 19]) + alias = a + a %= Array("uint8", [6, 7]) + + assert a is alias + assert alias.tolist() == [2, 5] + + +def test_array_fromfile_honours_current_position_for_eof(tmp_path) -> None: + from bitstring.array_ import Array + import pytest + + p = tmp_path / "fromfile_eof.bin" + p.write_bytes(bytes([1, 2, 3])) + + with p.open("rb") as f: + f.seek(2) + a = Array("uint8") + with pytest.raises(EOFError): + a.fromfile(f, 2) + assert a.tolist() == [3] diff --git a/tests/test_mxfp.py b/tests/test_mxfp.py index ce2d7053..59ba8973 100644 --- a/tests/test_mxfp.py +++ b/tests/test_mxfp.py @@ -2,9 +2,10 @@ import sys import math -from bitstring import BitArray, Dtype, Array, options +from bitstring import BitArray, Dtype, Array, CreationError import pytest import gfloat +from gfloat.formats import format_info_ocp_e4m3, format_info_ocp_e5m2 sys.path.insert(0, '..') @@ -17,40 +18,49 @@ def test_creation_e3m2mxfp(): assert len(x) == 6 def test_getting_e4m3mxfp_values(): - assert BitArray('0b00000000').e4m3mxfp == 0.0 - assert BitArray('0b01111110').e4m3mxfp == 448.0 - assert BitArray('0b11111110').e4m3mxfp == -448.0 - assert BitArray('0b00001000').e4m3mxfp == 2**-6 - assert math.isnan(BitArray('0b01111111').e4m3mxfp) - assert math.isnan(BitArray('0b11111111').e4m3mxfp) + assert BitArray('0b00000000').e4m3mxfp_saturate == 0.0 + assert BitArray('0b01111110').e4m3mxfp_saturate == 448.0 + assert BitArray('0b11111110').e4m3mxfp_saturate == -448.0 + assert BitArray('0b00001000').e4m3mxfp_saturate == 2**-6 + assert math.isnan(BitArray('0b01111111').e4m3mxfp_saturate) + assert math.isnan(BitArray('0b11111111').e4m3mxfp_saturate) def test_setting_e4m3mxfp_values(): x = BitArray('0b00000000') - x.e4m3mxfp = 0.0 + x.e4m3mxfp_saturate = 0.0 assert x == '0b00000000' - x.e4m3mxfp = -(2 ** -9) + x.e4m3mxfp_saturate = -(2 ** -9) assert x == '0b10000001' - x.e4m3mxfp = 99999 - assert x.e4m3mxfp == 448.0 + x.e4m3mxfp_saturate = 99999 + assert x.e4m3mxfp_saturate == 448.0 def test_getting_e5m2mxfp_values(): - assert BitArray('0b00000000').e5m2mxfp == 0.0 - assert BitArray('0b01111011').e5m2mxfp == 57344 + assert BitArray('0b00000000').e5m2mxfp_saturate == 0.0 + assert BitArray('0b01111011').e5m2mxfp_saturate == 57344 for b in ['0b01111101', '0b11111101', '0b01111110', '0b11111110', '0b01111111', '0b11111111']: - assert math.isnan(BitArray(b).e5m2mxfp) - assert BitArray('0b01111100').e5m2mxfp == float('inf') - assert BitArray('0b11111100').e5m2mxfp == float('-inf') - assert BitArray('0b00000001').e5m2mxfp == 2 ** -16 - assert BitArray('0b00000100').e5m2mxfp == 2 ** -14 + assert math.isnan(BitArray(b).e5m2mxfp_saturate) + assert BitArray('0b01111100').e5m2mxfp_saturate == float('inf') + assert BitArray('0b11111100').e5m2mxfp_saturate == float('-inf') + assert BitArray('0b00000001').e5m2mxfp_saturate == 2 ** -16 + assert BitArray('0b00000100').e5m2mxfp_saturate == 2 ** -14 def test_setting_e5m2mxfp_values(): x = BitArray('0b00000000') - x.e5m2mxfp = 0.0 + x.e5m2mxfp_saturate = 0.0 assert x == '0b00000000' - x.e5m2mxfp = -(2 ** -16) + x.e5m2mxfp_saturate = -(2 ** -16) assert x == '0b10000001' - x.e5m2mxfp = 99999 - assert x.e5m2mxfp == 57344.0 + x.e5m2mxfp_saturate = 99999 + assert x.e5m2mxfp_saturate == 57344.0 + + +def test_unadorned_e4m3_and_e5m2_mxfp_names_removed(): + for name in ['e4m3mxfp', 'e5m2mxfp']: + with pytest.raises(ValueError): + _ = Dtype(name) + with pytest.raises(CreationError): + _ = BitArray(**{name: 0.0}) + assert not hasattr(BitArray('0x00'), name) def test_getting_e3m2mxfp_values(): @@ -171,7 +181,7 @@ def test_setting_scaled_array(): assert sa[:] == [0.0, 1.0, 2.0] def test_multiple_scaled_arrays(): - d = bytes(b'hello_everyone!') + d = b'hello_everyone!' s1 = Array(Dtype('e2m1mxfp', scale=1), d) s2 = Array(Dtype('e2m1mxfp', scale=2**10), d) s3 = Array(Dtype('e2m1mxfp', scale=2**-10), d) @@ -285,47 +295,45 @@ def test_conversion_to_e8m0(): def test_rounding_to_even(): # When rounding to even, the value chosen when two are equidistant should end in a zero bit. - x = BitArray(e4m3mxfp=21.0) - assert x.e4m3mxfp == 20.0 - a = BitArray(e4m3mxfp=20.0).bin - b = BitArray(e4m3mxfp=22.0).bin + x = BitArray(e4m3mxfp_saturate=21.0) + assert x.e4m3mxfp_saturate == 20.0 + a = BitArray(e4m3mxfp_saturate=20.0).bin + b = BitArray(e4m3mxfp_saturate=22.0).bin assert a[:-1] == b[:-1] assert a[-1] == '0' assert b[-1] == '1' - x = BitArray(e4m3mxfp=22.0) - assert x.e4m3mxfp == 22.0 - x = BitArray(e4m3mxfp=23.0) - assert x.e4m3mxfp == 24.0 - x = BitArray(e4m3mxfp=24.0) - assert x.e4m3mxfp == 24.0 - - x = BitArray(e4m3mxfp=-50) # Midway between -48 and -52 - assert x.e4m3mxfp == -48.0 # Rounds towards zero + x = BitArray(e4m3mxfp_saturate=22.0) + assert x.e4m3mxfp_saturate == 22.0 + x = BitArray(e4m3mxfp_saturate=23.0) + assert x.e4m3mxfp_saturate == 24.0 + x = BitArray(e4m3mxfp_saturate=24.0) + assert x.e4m3mxfp_saturate == 24.0 + + x = BitArray(e4m3mxfp_saturate=-50) # Midway between -48 and -52 + assert x.e4m3mxfp_saturate == -48.0 # Rounds towards zero assert x.bin[-1] == '0' - x = BitArray(e4m3mxfp=-54) # Midway between -52 and -56 - assert x.e4m3mxfp == -56.0 # Rounds away from zero + x = BitArray(e4m3mxfp_saturate=-54) # Midway between -52 and -56 + assert x.e4m3mxfp_saturate == -56.0 # Rounds away from zero assert x.bin[-1] == '0' def test_rounding_consistent_to_gfloat(): - for fi, dt in [[gfloat.formats.format_info_ocp_e4m3, Dtype('e4m3mxfp')], - [gfloat.formats.format_info_ocp_e5m2, Dtype('e5m2mxfp')]]: + for fi, dt in [[format_info_ocp_e4m3, Dtype('e4m3mxfp_saturate')], + [format_info_ocp_e5m2, Dtype('e5m2mxfp_saturate')]]: for i in range(1 << 16): - f = BitArray(uint=i, length=16).float - mine = dt.parse(dt.build(f)) + f = BitArray(u=i, length=16).float + mine = dt.unpack(dt.pack(f)) theirs = gfloat.round_float(fi, f, sat=True) if math.isnan(mine): assert math.isnan(theirs) else: assert mine == theirs -@pytest.mark.usefixtures('switch_to_overflow') def test_rounding_consistent_to_gfloat_with_overflow(): - assert options.mxfp_overflow == 'overflow' - for fi, dt in [[gfloat.formats.format_info_ocp_e4m3, Dtype('e4m3mxfp')], - [gfloat.formats.format_info_ocp_e5m2, Dtype('e5m2mxfp')]]: + for fi, dt in [[format_info_ocp_e4m3, Dtype('e4m3mxfp_overflow')], + [format_info_ocp_e5m2, Dtype('e5m2mxfp_overflow')]]: for i in range(1 << 16): - f = BitArray(uint=i, length=16).float - mine = dt.parse(dt.build(f)) + f = BitArray(u=i, length=16).float + mine = dt.unpack(dt.pack(f)) theirs = gfloat.round_float(fi, f, sat=False) if math.isnan(mine): assert math.isnan(theirs) @@ -334,9 +342,9 @@ def test_rounding_consistent_to_gfloat_with_overflow(): print(mine, theirs) def test_conversion_from_nan(): - x = BitArray(e4m3mxfp=float('nan')) + x = BitArray(e4m3mxfp_saturate=float('nan')) assert x == '0b11111111' - x = BitArray(e5m2mxfp=float('nan')) + x = BitArray(e5m2mxfp_saturate=float('nan')) assert x == '0b11111111' with pytest.raises(ValueError): _ = BitArray(e3m2mxfp=float('nan')) @@ -376,54 +384,50 @@ def test_conversion_from_inf(): _ = BitArray(e8m0mxfp=float('-inf')) def test_conversion_to_8bit_with_saturate(): - assert options.mxfp_overflow == 'saturate' - x = BitArray(e5m2mxfp=float('inf')) - assert x.e5m2mxfp == 57344.0 - x = BitArray(e5m2mxfp=float('-inf')) - assert x.e5m2mxfp == -57344.0 - - x = BitArray(e5m2mxfp=1e10) - assert x.e5m2mxfp == 57344.0 - x = BitArray(e5m2mxfp=-1e10) - assert x.e5m2mxfp == -57344.0 - - x = BitArray(e4m3mxfp=float('inf')) - assert x.e4m3mxfp == 448.0 - x = BitArray(e4m3mxfp=float('-inf')) - assert x.e4m3mxfp == -448.0 - - x = BitArray(e4m3mxfp=1e10) - assert x.e4m3mxfp == 448.0 - x = BitArray(e4m3mxfp=-1e10) - assert x.e4m3mxfp == -448.0 - -@pytest.fixture -def switch_to_overflow(): - options.mxfp_overflow = 'overflow' - yield - options.mxfp_overflow = 'saturate' - -@pytest.mark.usefixtures('switch_to_overflow') + x = BitArray(e5m2mxfp_saturate=float('inf')) + assert x.e5m2mxfp_saturate == 57344.0 + assert x == BitArray(e5m2mxfp_saturate=float('inf')) + x = BitArray(e5m2mxfp_saturate=float('-inf')) + assert x.e5m2mxfp_saturate == -57344.0 + + x = BitArray(e5m2mxfp_saturate=1e10) + assert x.e5m2mxfp_saturate == 57344.0 + x = BitArray(e5m2mxfp_saturate=-1e10) + assert x.e5m2mxfp_saturate == -57344.0 + + x = BitArray(e4m3mxfp_saturate=float('inf')) + assert x.e4m3mxfp_saturate == 448.0 + assert x == BitArray(e4m3mxfp_saturate=float('inf')) + x = BitArray(e4m3mxfp_saturate=float('-inf')) + assert x.e4m3mxfp_saturate == -448.0 + + x = BitArray(e4m3mxfp_saturate=1e10) + assert x.e4m3mxfp_saturate == 448.0 + x = BitArray(e4m3mxfp_saturate=-1e10) + assert x.e4m3mxfp_saturate == -448.0 + def test_conversion_to_8bit_with_overflow(): - x = BitArray(e5m2mxfp=float('inf')) - assert x.e5m2mxfp == float('inf') - x = BitArray(e5m2mxfp=float('-inf')) - assert x.e5m2mxfp == float('-inf') - - x = BitArray(e5m2mxfp=1e10) - assert x.e5m2mxfp == float('inf') - x = BitArray(e5m2mxfp=-1e10) - assert x.e5m2mxfp == float('-inf') - - x = BitArray(e4m3mxfp=float('inf')) - assert math.isnan(x.e4m3mxfp) - x = BitArray(e4m3mxfp=float('-inf')) - assert math.isnan(x.e4m3mxfp) - - x = BitArray(e4m3mxfp=1e10) - assert math.isnan(x.e4m3mxfp) - x = BitArray(e4m3mxfp=-1e10) - assert math.isnan(x.e4m3mxfp) + x = BitArray(e5m2mxfp_overflow=float('inf')) + assert x.e5m2mxfp_saturate == float('inf') + assert x.e5m2mxfp_overflow == float('inf') + x = BitArray(e5m2mxfp_overflow=float('-inf')) + assert x.e5m2mxfp_saturate == float('-inf') + + x = BitArray(e5m2mxfp_overflow=1e10) + assert x.e5m2mxfp_saturate == float('inf') + x = BitArray(e5m2mxfp_overflow=-1e10) + assert x.e5m2mxfp_saturate == float('-inf') + + x = BitArray(e4m3mxfp_overflow=float('inf')) + assert math.isnan(x.e4m3mxfp_saturate) + assert math.isnan(x.e4m3mxfp_overflow) + x = BitArray(e4m3mxfp_overflow=float('-inf')) + assert math.isnan(x.e4m3mxfp_saturate) + + x = BitArray(e4m3mxfp_overflow=1e10) + assert math.isnan(x.e4m3mxfp_saturate) + x = BitArray(e4m3mxfp_overflow=-1e10) + assert math.isnan(x.e4m3mxfp_saturate) def test_mxint_rounding(): x = BitArray('mxint=0.0') @@ -452,15 +456,14 @@ def test_mxint_rounding(): x.mxint = -3.5 / 64.0 assert x.mxint == -4.0 / 64.0 -@pytest.mark.usefixtures('switch_to_overflow') -@pytest.mark.parametrize("fmt", [Dtype(x) for x in ['e5m2mxfp', 'e4m3mxfp', 'e2m3mxfp', 'e3m2mxfp', 'e2m1mxfp', 'e8m0mxfp', 'mxint', 'p3binary', 'p4binary']]) +@pytest.mark.parametrize("fmt", [Dtype(x) for x in ['e5m2mxfp_overflow', 'e4m3mxfp_overflow', 'e2m3mxfp', 'e3m2mxfp', 'e2m1mxfp', 'e8m0mxfp', 'mxint', 'p3binary', 'p4binary']]) def test_roundtrips(fmt): - # Note that e5m2mxfp in saturate mode will not pass this test, as inf -> inf -> max_value. + # Note that e5m2mxfp_saturate in saturate mode will not pass this test, as inf -> inf -> max_value. for i in range(1 << fmt.bitlength): b = BitArray(u=i, length=fmt.bitlength) - v = fmt.parse(b) - b2 = fmt.build(v) - if math.isnan(fmt.parse(b)): - assert math.isnan(fmt.parse(b2)) + v = fmt.unpack(b) + b2 = fmt.pack(v) + if math.isnan(fmt.unpack(b)): + assert math.isnan(fmt.unpack(b2)) else: assert b == b2 diff --git a/tests/test_reader.py b/tests/test_reader.py new file mode 100644 index 00000000..9e082ed4 --- /dev/null +++ b/tests/test_reader.py @@ -0,0 +1,184 @@ +import pytest + +import bitstring +from bitstring import BitArray, Bits, Dtype, Reader, pack + + +def test_creation_and_bits_property(): + bits = Bits("0x12345") + r = Reader(bits, 4) + assert r.bits is bits + assert len(r) == 20 + assert r.pos == 4 + assert repr(r) == "Reader(, pos=4)" + + mutable_bits = BitArray("0xff") + r.bits = mutable_bits + assert r.bits is mutable_bits + assert r.pos == 4 + + with pytest.raises(TypeError): + Reader("0x234") + with pytest.raises(TypeError): + Reader(b"hello") + with pytest.raises(TypeError): + r.bits = "0xff" + + +def test_lax_position_storage(): + r = Reader(Bits("0xf"), pos=100) + assert r.pos == 100 + with pytest.raises(ValueError): + r.read(1) + assert r.pos == 100 + + r.pos = -10 + assert r.bitpos == -10 + with pytest.raises(ValueError): + r.read(1) + assert r.pos == -10 + + r.bytepos = 6 + assert r.pos == 48 + assert r.bytepos == 6 + + +def test_bytepos_requires_alignment(): + r = Reader(Bits("0xff"), pos=3) + with pytest.raises(bitstring.ByteAlignError): + _ = r.bytepos + r.pos = 8 + assert r.bytepos == 1 + + +def test_read_and_peek(): + r = Reader(Bits("0x12345"), 4) + assert r.read(4) == "0x2" + assert r.pos == 8 + assert r.read("u4") == 3 + assert r.pos == 12 + assert r.peek(Dtype("hex4")) == "4" + assert r.pos == 12 + assert r.read("hex") == "45" + assert r.pos == 20 + + +def test_read_failure_restores_pos(): + r = Reader(Bits("0xff"), 4) + with pytest.raises(bitstring.ReadError): + r.read(5) + assert r.pos == 4 + + with pytest.raises(bitstring.ReadError): + r.read("uint5") + assert r.pos == 4 + + with pytest.raises(bitstring.ReadError): + r.peek("uint5") + assert r.pos == 4 + + +def test_read_list_and_peek_list(): + r = Reader(Bits("0b10001111001")) + assert r.read_list("pad:1, uint:3, pad:4, uint:3") == [0, 1] + assert r.pos == 11 + + r.pos = 0 + assert r.read_list(["5", "3", 3]) == [Bits("0b10001"), Bits("0b111"), Bits("0b001")] + assert r.pos == 11 + + r = Reader(Bits("0x0102")) + assert r.peek_list("bits8, hex:b", b=4) == [Bits("0x01"), "0"] + assert r.pos == 0 + + +def test_read_to(): + r = Reader(Bits("0xaabb")) + assert r.read_to("0xaa", bytealigned=True) == "0xaa" + assert r.bytepos == 1 + + r = Reader(Bits("0xaabb00aa00bb")) + assert r.read_to("0x00", bytealigned=True) == "0xaabb00" + assert r.bytepos == 3 + assert r.read_to("0xaa", bytealigned=True) == "0xaa" + + old_pos = r.pos + with pytest.raises(bitstring.ReadError): + r.read_to("0xcc", bytealigned=True) + assert r.pos == old_pos + + with pytest.raises(ValueError): + r.read_to(4) + + +def test_byte_align(): + r = Reader(Bits("0xabcdef"), pos=3) + assert r.byte_align() == 5 + assert r.pos == 8 + assert r.byte_align() == 0 + + r = Reader(Bits("0b1"), pos=1) + with pytest.raises(ValueError): + r.byte_align() + assert r.pos == 1 + + +def test_old_reader_method_names_still_work(): + r = Reader(Bits("0b10001111001")) + assert r.readlist("pad:1, uint:3, pad:4, uint:3") == [0, 1] + r.pos = 0 + assert r.peeklist("bits5") == [Bits("0b10001")] + + r = Reader(Bits("0xaabb"), pos=3) + assert r.bytealign() == 5 + assert r.readto("0xbb", bytealigned=True) == "0xbb" + + +def test_find_and_rfind_update_pos_only_on_success(): + r = Reader(Bits("0b000111000111"), pos=5) + assert r.find("0b111") == 3 + assert r.pos == 3 + assert r.find("0b101") is None + assert r.pos == 3 + assert r.rfind("0b111") == 9 + assert r.pos == 9 + + +@pytest.mark.parametrize( + "call", + [ + lambda r: r.find("0b1", 0), + lambda r: r.rfind("0b1", 0), + lambda r: r.readto("0b1", False), + ], +) +def test_reader_optional_search_arguments_are_keyword_only(call): + with pytest.raises(TypeError): + call(Reader(Bits("0b1010"))) + + +def test_mutable_bits_are_exposed_directly(): + bits = BitArray("0x00ff") + r = Reader(bits) + assert r.bits is bits + assert r.read("u8") == 0 + assert r.pos == 8 + + r.bits.append("0xff") + assert bits == "0x00ffff" + assert r.pos == 8 + assert r.read("u8") == 255 + + bits.clear() + assert r.pos == 16 + with pytest.raises(ValueError): + r.read(1) + assert r.pos == 16 + + +def test_reader_with_pack_result(): + bits = pack("uint8, uint8", 1, 2) + assert type(bits) is Bits + r = Reader(bits) + assert r.read("uint8") == 1 + assert r.read("uint8") == 2 diff --git a/tests/test_reader_legacy.py b/tests/test_reader_legacy.py new file mode 100644 index 00000000..309bb14c --- /dev/null +++ b/tests/test_reader_legacy.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python + +import io +import os +import platform + +import pytest + +import bitstring +from bitstring import BitArray, Bits, Reader + + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def test_from_file(): + s = Reader(Bits(filename=os.path.join(THIS_DIR, 'test.m1v'))) + assert s.bits[0:32].hex == '000001b3' + assert s.read(8 * 4).hex == '000001b3' + width = s.read(12).uint + height = s.read(12).uint + assert (width, height) == (352, 288) + + +def test_from_file_with_offset_and_length(): + s = Reader(Bits(filename=os.path.join(THIS_DIR, 'test.m1v'), offset=24, length=8)) + assert s.bits.hex == 'b3' + reconstructed = '' + for bit in s.bits: + reconstructed += '1' if bit is True else '0' + assert reconstructed == s.bits.bin + + +def test_interleaved_exp_golomb_reading(): + s = Reader(Bits(uie=333)) + assert s.read('uie') == 333 + u = Bits('uie=12, sie=-9, sie=9, uie=1000000').unpack('uie, 2*sie, uie') + assert u == [12, -9, 9, 1000000] + + +def test_interleaved_exp_golomb_reading_errors_restore_position(): + s = Reader(Bits.from_zeros(10)) + with pytest.raises(bitstring.ReadError): + s.read('uie') + assert s.pos == 0 + with pytest.raises(bitstring.ReadError): + s.read('sie') + assert s.pos == 0 + + +def test_readto_byte_aligned(): + a = Reader(Bits('0xaabb00aa00bb')) + b = a.readto('0x00', bytealigned=True) + assert b == '0xaabb00' + assert a.bytepos == 3 + b = a.readto('0xaa', bytealigned=True) + assert b == '0xaa' + with pytest.raises(bitstring.ReadError): + Reader(b).readto('0xcc', bytealigned=True) + + +def test_readto_not_aligned(): + a = Reader(Bits('0b00111001001010011011'), pos=1) + assert a.readto('0b00') == '0b011100' + assert a.readto('0b110') == '0b10010100110' + with pytest.raises(ValueError): + a.readto('') + + +def test_readto_disallows_integers(): + a = Reader(Bits('0x0f')) + with pytest.raises(ValueError): + a.readto(4) + + +def test_reading_lines(): + s = b"This is a test\nof reading lines\nof text\n" + b = Reader(Bits(bytes=s)) + n = Bits(bytes=b'\n') + assert b.readto(n).bytes == b'This is a test\n' + assert b.readto(n).bytes == b'of reading lines\n' + assert b.readto(n).bytes == b'of text\n' + + +def test_pad_token_read(): + s = Reader(Bits('0b100011110001')) + assert s.read('pad:1') is None + assert s.pos == 1 + assert s.read(3) == Bits('0b000') + assert s.read('pad:1') is None + assert s.pos == 5 + + +def test_pad_token_readlist(): + s = Reader(Bits.from_string('0b10001111001')) + t = s.readlist('pad:1, uint:3, pad:4, uint:3') + assert t == [0, 1] + s.pos = 0 + t = s.readlist('pad:1, pad:5') + assert t == [] + assert s.pos == 6 + s.pos = 0 + t = s.readlist('pad:1, bin, pad:4, uint:3') + assert t == ['000', 1] + s.pos = 0 + t = s.readlist('pad, bin:3, pad:4, uint:3') + assert t == ['000', 1] + + +def test_unpacking_bytes(): + s = Bits.from_zeros(80) + t = s.unpack('bytes:1') + assert t[0] == b'\x00' + a, b, c = s.unpack('bytes:1, bytes, bytes2') + assert a == b'\x00' + assert b == b'\x00' * 7 + assert c == b'\x00' * 2 + + +def test_unpacking_bytes_with_keywords(): + s = Bits('0x55' * 10) + t = s.unpack('pad:a, bytes:b, bytes, pad:a', a=4, b=6) + assert t == [b'\x55' * 6, b'\x55' * 3] + + +def test_read_bits_as_default(): + s = Reader(Bits('uint:31=14')) + v = s.read(31) + assert v.uint == 14 + s.pos = 0 + + +def test_readlist_bits_as_default(): + s = Reader(Bits('uint:5=3, uint:3=0, uint:11=999')) + v = s.readlist([5, 3, 11]) + assert [x.uint for x in v] == [3, 0, 999] + s.pos = 0 + v = s.readlist(['5', '3', 11]) + assert [x.uint for x in v] == [3, 0, 999] + + +def test_bytesio_creation(): + f = io.BytesIO(b"\x12\xff\x77helloworld") + s = Bits.from_bytes(f.getvalue()) + assert s[0:8] == '0x12' + assert len(s) == 13 * 8 + s = Bits.from_bytes(f.getvalue(), offset=8, length=12) + assert s == '0xff7' + + +def test_bytesio_creation_exceptions(): + f = io.BytesIO(b"123456789") + _ = Bits.from_bytes(f.getvalue(), length=9 * 8) + with pytest.raises(bitstring.CreationError): + _ = Bits.from_bytes(f.getvalue(), length=9 * 8 + 1) + with pytest.raises(bitstring.CreationError): + _ = Bits.from_bytes(f.getvalue(), length=9 * 8, offset=1) + + +def test_reader_default_pos(): + s = Reader(Bits('0xabc')) + assert s.pos == 0 + + +def test_reader_lax_positive_pos(): + s = Reader(Bits('0xabc'), pos=0) + assert s.pos == 0 + s = Reader(Bits('0xabc'), pos=1) + assert s.pos == 1 + s = Reader(Bits('0xabc'), pos=12) + assert s.pos == 12 + s = Reader(Bits('0xabc'), pos=13) + assert s.pos == 13 + with pytest.raises(ValueError): + s.read(0) + + +def test_reader_lax_negative_pos(): + s = Reader(Bits('0xabc'), pos=-1) + assert s.pos == -1 + with pytest.raises(ValueError): + s.read(1) + s = Reader(Bits('0xabc'), pos=-12) + assert s.pos == -12 + + +def test_reader_string_representation(): + s = Reader(Bits('0b110'), pos=2) + assert repr(s) == "Reader(, pos=2)" + + +def test_reader_string_representation_from_file(): + filename = os.path.join(THIS_DIR, 'test.m1v') + s = Reader(Bits(filename=filename), pos=2001) + assert repr(s) == "Reader(, pos=2001)" + s.pos = 0 + assert repr(s) == "Reader(, pos=0)" + + +def test_windows_file_lock_bug(tmp_path): + path = tmp_path / 'temp_unit_test_file' + with open(path, mode='w') as f: + f.write('Hello') + _ = Bits(filename=path) + + try: + with open(path, mode='w') as _: + pass + except OSError: + if platform.system() == 'Windows': + pass + + +def test_readerrors(): + s = Reader(Bits('0b110')) + s.read(3) + with pytest.raises(bitstring.ReadError): + _ = s.read(1) + s.pos = 1 + with pytest.raises(bitstring.ReadError): + _ = s.read('u3') + + +def test_mutable_reader_uses_external_bitarray_without_position_coupling(): + bits = BitArray('0x001122') + r = Reader(bits, pos=8) + assert r.read('uint8') == 0x11 + bits.insert('0xff', 8) + assert bits == '0x00ff1122' + assert r.pos == 16 + r.pos = 8 + assert r.read('uint8') == 0xff diff --git a/tests/test_scaled_dtypes.py b/tests/test_scaled_dtypes.py index af5abb68..07e5e9b3 100644 --- a/tests/test_scaled_dtypes.py +++ b/tests/test_scaled_dtypes.py @@ -9,5 +9,5 @@ def test_simple_scaled_dtype(): d = Dtype('u4', scale=2) assert d.scale == 2 b = BitArray('0x01') - assert d.parse(b) == 2 + assert d.unpack(b) == 2