From e7226a538c161836930727922155a7b4ab3514ad Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Fri, 7 Aug 2026 22:01:07 +0000 Subject: [PATCH 1/2] Add open access flag --- Readme.md | 28 +++- pyproject.toml | 2 +- setup.cfg | 2 +- src/hecdss/__init__.py | 1 + src/hecdss/download_hecdss.py | 9 +- src/hecdss/dss_access.py | 32 ++++ src/hecdss/hecdss.py | 48 +++++- src/hecdss/native.py | 36 ++++- tests/test_access.py | 278 ++++++++++++++++++++++++++++++++++ 9 files changed, 421 insertions(+), 15 deletions(-) create mode 100644 src/hecdss/dss_access.py create mode 100644 tests/test_access.py diff --git a/Readme.md b/Readme.md index 7399f33..3dc5ddd 100644 --- a/Readme.md +++ b/Readme.md @@ -15,7 +15,7 @@ pip install hecdss [API Documentation](https://hydrologicengineeringcenter.github.io/hec-dss-python/) ### DSS file methods -1. `HecDss(file_path: str)`: Opens a DSS file located at the provided file path. +1. `HecDss(file_path: str, access: DssAccess = DssAccess.GENERAL_ACCESS)`: Opens a DSS file located at the provided file path. See [File Access Modes](#file-access-modes). 2. `get(record_path: str)`: Retrieves the record data from the currently opened DSS file of the designated path. @@ -150,6 +150,32 @@ This library uses (`hecdss.dll` on Windows and `libhecdss.so` on Unix/Linux). h ``` +## File Access Modes + +`HecDss` takes an optional `access` argument that controls how the DSS file is opened. These +modes map directly to the `access` argument of `hec_dss_open_ex` in the HEC-DSS C library. + +| `DssAccess` | Value | Description | +| ----------- | ----- | ----------- | +| `GENERAL_ACCESS` | 0 | Read, or read/write when the file allows it. No error when the file does not have write permission. This is the default. | +| `READ_ACCESS` | 1 | Read only. The file must already exist and is never written to, so several processes can read the same file at the same time. | +| `MULTI_USER_ACCESS` | 2 | Read/write with full multi-user access. Usually slow, but necessary when more than one process writes to the file. | +| `SINGLE_USER_ADVISORY_ACCESS` | 3 | Read/write with multi-user advisory access. Errors when the file is read only. | +| `EXCLUSIVE_ACCESS` | 4 | Exclusive write, used for squeezing. Errors when exclusive access is not available. | + +```python +from hecdss import DssAccess, HecDss + +# open a file read only; several processes can do this at the same time +with HecDss("example.dss", DssAccess.READ_ACCESS) as dss: + data = dss.get("/example/data/////") + print(dss.access, dss.readonly) # DssAccess.READ_ACCESS True +``` + +A file opened with `READ_ACCESS` must already exist; opening a missing file raises `FileNotFoundError` +instead of creating it. Methods that modify the file (`put`, `delete`, `writePrecompressedGrid`) +raise `PermissionError` on a `READ_ACCESS` file. + ## Message Levels Message levels can be set using `set_global_debug_level` or `set_debug_level`, for example: diff --git a/pyproject.toml b/pyproject.toml index f166f18..287dd5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hec-dss-python" -version = "0.1.32" +version = "0.1.33" description = "Python wrapper for the HEC-DSS file database C library." authors = ["Hydrologic Engineering Center"] license = "MIT" diff --git a/setup.cfg b/setup.cfg index 9eac351..51e5651 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = hecdss -version = 0.1.32 +version = 0.1.33 author = Hydrologic Engineering Center author_email =hec.dss@usace.army.mil description = Python wrapper for the HEC-DSS file database C library. diff --git a/src/hecdss/__init__.py b/src/hecdss/__init__.py index 65c231e..225afcc 100644 --- a/src/hecdss/__init__.py +++ b/src/hecdss/__init__.py @@ -1,5 +1,6 @@ from hecdss.catalog import Catalog +from hecdss.dss_access import DssAccess from hecdss.hecdss import HecDss from hecdss.dsspath import DssPath from hecdss.irregular_timeseries import IrregularTimeSeries diff --git a/src/hecdss/download_hecdss.py b/src/hecdss/download_hecdss.py index a385256..d275153 100644 --- a/src/hecdss/download_hecdss.py +++ b/src/hecdss/download_hecdss.py @@ -1,10 +1,11 @@ """Helper module to retrieve the binary libraries""" -from pathlib import Path +import os import shutil -import requests import zipfile -import os +from pathlib import Path + +import requests def download_and_unzip(url, zip_file, destination_dir): @@ -36,7 +37,7 @@ def download_and_unzip(url, zip_file, destination_dir): print(f"Failed to download zip file. Status code: {response.status_code}") base_url = "https://www.hec.usace.army.mil/nexus/repository/maven-public/mil/army/usace/hec/hecdss/" -version = "7-JA-8" +version = "7-JA-9" destination_dir = Path(__file__).parent.joinpath("lib") zip_url = f"{base_url}{version}-win-x86_64/hecdss-{version}-win-x86_64.zip" diff --git a/src/hecdss/dss_access.py b/src/hecdss/dss_access.py new file mode 100644 index 0000000..1fc0b56 --- /dev/null +++ b/src/hecdss/dss_access.py @@ -0,0 +1,32 @@ +from enum import IntEnum + + +class DssAccess(IntEnum): + """DssAccess is an enumeration of the ways a DSS file can be opened + + The values match the access argument of hec_dss_open_ex in the HEC-DSS C + library. + + Returns: + DssAccess: the access mode + """ + + GENERAL_ACCESS = 0 + """Read or read/write, whichever the file allows. No error when the file + does not have write permission. This is the default.""" + + READ_ACCESS = 1 + """Read only; the file will not be written to, and must already exist. + Several processes may hold the same file open this way at once.""" + + MULTI_USER_ACCESS = 2 + """Read/write with full multi-user access. Usually slow, but necessary when + more than one process writes to the file at the same time.""" + + SINGLE_USER_ADVISORY_ACCESS = 3 + """Read/write with multi-user advisory access. Errors when the file is read + only.""" + + EXCLUSIVE_ACCESS = 4 + """Exclusive write, used for squeezing. Errors when exclusive access is not + available.""" diff --git a/src/hecdss/hecdss.py b/src/hecdss/hecdss.py index fecaaac..0d3b981 100644 --- a/src/hecdss/hecdss.py +++ b/src/hecdss/hecdss.py @@ -8,6 +8,7 @@ from hecdss.array_container import ArrayContainer from hecdss.catalog import Catalog from hecdss.dateconverter import DateConverter +from hecdss.dss_access import DssAccess from hecdss.dsspath import DssPath from hecdss.gridded_data import GriddedData from hecdss.irregular_timeseries import IrregularTimeSeries @@ -27,19 +28,52 @@ class HecDss: """ Main class for working with DSS files """ - def __init__(self, filename: str): + def __init__(self, filename: str, access: DssAccess = DssAccess.GENERAL_ACCESS): """constructor for HecDSS Args: - filename (str): DSS filename to be opened; it will be created if it doesn't exist. + filename (str): DSS filename to be opened; it will be created if it + doesn't exist, unless access is DssAccess.READ_ACCESS. + access (DssAccess): how the file is opened. dss_access.py defines access modes + + Raises: + ValueError: access is not one of the DssAccess values. + FileNotFoundError: access is READ_ACCESS and the file does not exist. """ + self._access = DssAccess(access) self._native = _Native() - self._native.hec_dss_open(filename) + self._native.hec_dss_open(filename, self._access) self._catalog = None self._filename = filename self._closed = False + @property + def access(self) -> DssAccess: + """DssAccess: the access mode this file was opened with""" + return self._access + + @property + def readonly(self) -> bool: + """bool: True when this file was opened with DssAccess.READ_ACCESS""" + return self._access == DssAccess.READ_ACCESS + + def _check_writable(self, operation: str) -> None: + """raises if this file was opened read only + + Args: + operation (str): name of the method being attempted, used in the message. + + Raises: + PermissionError: the file was opened with DssAccess.READ_ACCESS. + """ + if self.readonly: + raise PermissionError( + f"cannot {operation}: '{self._filename}' was opened with " + f"{self._access.name}. Reopen it with a read/write access mode " + "to modify it." + ) + def __enter__(self): """ Enter the runtime context related to this object. @@ -609,10 +643,12 @@ def put(self, container) -> int: Raises: NotImplementedError: if saving the type of container is not supported. + PermissionError: if the file was opened with DssAccess.READ_ACCESS. Returns: int: status of zero when successful. Non zero for errors. """ + self._check_writable("put") # TODO. is timezone needed? status = 0 if type(container) is RegularTimeSeries: @@ -712,9 +748,12 @@ def writePrecompressedGrid(self, gd, compressedData, CompressionSize): Args compressedData (bytes): Compressed data. CompressionSize (int): Size of the compressed data. + Raises: + PermissionError: if the file was opened with DssAccess.READ_ACCESS. Returns: int: 0 if successful, -1 otherwise. """ + self._check_writable("writePrecompressedGrid") if compressedData and CompressionSize > 0: status = self._native.hec_dss_gridStore(gd, compressedData, CompressionSize) @@ -729,9 +768,12 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en allRecords (bool): if True, delete all records with this pathname that have different dates startdatetime (datetime): start date for query, if only start date is provided, delete all records at or after this date enddatetime (datetime): end date for the query, if only end date is provided, delete all records at or before this date + Raises: + PermissionError: if the file was opened with DssAccess.READ_ACCESS. Returns: int: status of zero when successful. Non zero for errors. """ + self._check_writable("delete") rt = self.get_record_type(pathname) delete_path = DssPath(pathname) if (rt == RecordType.RegularTimeSeries or rt == RecordType.IrregularTimeSeries) and allrecords: diff --git a/src/hecdss/native.py b/src/hecdss/native.py index 9d91e6e..af5a1c7 100644 --- a/src/hecdss/native.py +++ b/src/hecdss/native.py @@ -19,6 +19,8 @@ import numpy as np +from hecdss.dss_access import DssAccess + # from hecdss.location_info import LocationInfo @@ -76,25 +78,49 @@ def __init__(self): else: self.dll = self.load_hecdss_library("libhecdss.so") - def hec_dss_open(self, dss_filename: str) -> int: + def hec_dss_open(self, dss_filename: str, access: int = DssAccess.GENERAL_ACCESS) -> int: """opens a DSS file and gets a handle Args: - dss_filename (str): filename to open + dss_filename (str): filename to open; it is created if it doesn't + exist, except when access is READ_ACCESS. + access (int): read/write access used to open the file, see DssAccess + 0 - GENERAL_ACCESS: Doesn't matter (no error if file doesn't have write permission) + 1 - READ_ACCESS: Read only (will not allow writing to file) + 2 - MULTI_USER_ACCESS: Read/Write permission with full multi-user access + (usually slow, but necessary for multiple processes) + 3 - SINGLE_USER_ADVISORY_ACCESS: Read/Write permission with multi-user advisory + access (throws an error if file is read only) + 4 - EXCLUSIVE_ACCESS: Exclusive write (used for squeezing). + Throws an error if not available. Returns: int: status of zero when successful, non-zero on error. + + Raises: + ValueError: access is not one of the DssAccess values. + FileNotFoundError: READ_ACCESS was requested and the file is missing. + Exception: the file could not be opened. """ - f = self.dll.hec_dss_open + access = DssAccess(access) + + if access == DssAccess.READ_ACCESS and not os.path.exists(dss_filename): + raise FileNotFoundError( + f"DSS file not found: '{dss_filename}'. " + f"A file must already exist to be opened with {access.name}." + ) + + f = self.dll.hec_dss_open_ex f.argtypes = [ c_char_p, POINTER(c_void_p), + c_int, ] f.restype = c_int self.handle = c_void_p() - rval = f(dss_filename.encode("utf-8"), ctypes.byref(self.handle)) + rval = f(dss_filename.encode("utf-8"), byref(self.handle), c_int(access)) if rval != 0: - raise Exception("Error opening DSS file.") + raise Exception(f"Error opening DSS file. status = {rval}") return rval def hec_dss_close(self): diff --git a/tests/test_access.py b/tests/test_access.py new file mode 100644 index 0000000..74d9b06 --- /dev/null +++ b/tests/test_access.py @@ -0,0 +1,278 @@ + +import hashlib +import multiprocessing +import os +import unittest +from datetime import datetime + +from file_manager import FileManager + +from hecdss import DssAccess, HecDss + + +PATHNAME = "//SACRAMENTO/PRECIP-INC//1Day/OBS/" +START = datetime(2005, 1, 1) +END = datetime(2005, 1, 4) + +READ_WRITE_ACCESS = [ + DssAccess.GENERAL_ACCESS, + DssAccess.MULTI_USER_ACCESS, + DssAccess.SINGLE_USER_ADVISORY_ACCESS, + DssAccess.EXCLUSIVE_ACCESS, +] + +PROCESS_COUNT = 4 +READS_PER_PROCESS = 25 + +MP_TIMEOUT_SECONDS = 120 + + +def _file_fingerprint(filename): + """Returns (size, sha256) of a file, used to prove readers do not write.""" + with open(filename, "rb") as f: + return os.path.getsize(filename), hashlib.sha256(f.read()).hexdigest() + + +def _read_once(filename): + """Opens filename read-only and returns the values of PATHNAME.""" + HecDss.set_global_debug_level(0) + with HecDss(filename, DssAccess.READ_ACCESS) as dss: + return list(dss.get(PATHNAME, START, END).values) + + +def _read_repeatedly(args): + """Opens filename read-only once and reads PATHNAME repeatedly. + + Returns (pid, number_of_reads_that_returned_the_expected_values). + """ + filename, iterations, expected = args + HecDss.set_global_debug_level(0) + matches = 0 + with HecDss(filename, DssAccess.READ_ACCESS) as dss: + for _ in range(iterations): + if list(dss.get(PATHNAME, START, END).values) == expected: + matches += 1 + return os.getpid(), matches + + +def _run_workers(func, arglist): + """Runs func over arglist in separate spawned processes and returns results.""" + ctx = multiprocessing.get_context("spawn") + with ctx.Pool(len(arglist)) as pool: + return pool.map_async(func, arglist).get(timeout=MP_TIMEOUT_SECONDS) + + +class TestAccessModes(unittest.TestCase): + """Opening a file with each of the access modes of hec_dss_open_ex.""" + + def setUp(self) -> None: + self.test_files = FileManager() + HecDss.set_global_debug_level(0) + + def tearDown(self) -> None: + self.test_files.cleanup() + + def test_every_access_mode_reads_the_same_data(self): + filename = self.test_files.get_copy("sample7.dss") + + with HecDss(filename) as dss: + expected = list(dss.get(PATHNAME, START, END).values) + self.assertTrue(expected, "test fixture returned no values") + + for access in DssAccess: + with self.subTest(access=access.name): + # a fresh copy per mode, so a writable mode cannot affect the next + filename = self.test_files.get_copy("sample7.dss") + with HecDss(filename, access) as dss: + self.assertEqual(expected, list(dss.get(PATHNAME, START, END).values)) + + def test_default_access_is_general_access(self): + filename = self.test_files.get_copy("sample7.dss") + + with HecDss(filename) as dss: + self.assertEqual(DssAccess.GENERAL_ACCESS, dss.access) + self.assertFalse(dss.readonly) + + def test_access_is_reported_and_plain_ints_are_accepted(self): + filename = self.test_files.get_copy("sample7.dss") + + with HecDss(filename, 1) as dss: + self.assertEqual(DssAccess.READ_ACCESS, dss.access) + self.assertTrue(dss.readonly) + + def test_invalid_access_raises_value_error(self): + filename = self.test_files.get_copy("sample7.dss") + + for access in (-1, 5, "read"): + with self.subTest(access=access): + with self.assertRaises(ValueError): + HecDss(filename, access) + + def test_read_write_modes_can_write(self): + for access in READ_WRITE_ACCESS: + with self.subTest(access=access.name): + filename = self.test_files.get_copy("sample7.dss") + + with HecDss(filename, access) as dss: + self.assertFalse(dss.readonly) + ts = dss.get(PATHNAME, START, END) + expected = list(ts.values * 2) + ts.values = ts.values * 2 + self.assertEqual(0, dss.put(ts)) + + with HecDss(filename, DssAccess.READ_ACCESS) as dss: + self.assertEqual(expected, list(dss.get(PATHNAME, START, END).values)) + + +class TestReadAccess(unittest.TestCase): + """Read-only behavior in a single process.""" + + def setUp(self) -> None: + self.test_files = FileManager() + HecDss.set_global_debug_level(0) + + def tearDown(self) -> None: + self.test_files.cleanup() + + def test_readonly_catalog_and_record_count(self): + """Read paths other than get() also work on a read-only handle.""" + filename = self.test_files.get_copy("sample7.dss") + + with HecDss(filename) as dss: + expected_count = dss.record_count() + + with HecDss(filename, DssAccess.READ_ACCESS) as dss: + self.assertEqual(expected_count, dss.record_count()) + self.assertTrue(len(dss.get_catalog().uncondensed_paths) > 0) + + def test_readonly_missing_file_raises_and_does_not_create_it(self): + missing = self.test_files.create_test_file(".dss") + self.assertFalse(os.path.exists(missing)) + + with self.assertRaises(FileNotFoundError): + HecDss(missing, DssAccess.READ_ACCESS) + + self.assertFalse( + os.path.exists(missing), + "read-only open must not create the file", + ) + + def test_readwrite_still_creates_missing_file(self): + """The default path is unchanged by the access argument.""" + filename = self.test_files.create_test_file(".dss") + + with HecDss(filename) as dss: + self.assertEqual(0, dss.record_count()) + + self.assertTrue(os.path.exists(filename)) + + def test_put_on_readonly_raises(self): + filename = self.test_files.get_copy("sample7.dss") + + with HecDss(filename) as dss: + container = dss.get(PATHNAME, START, END) + + with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with self.assertRaises(PermissionError): + dss.put(container) + + def test_delete_on_readonly_raises(self): + filename = self.test_files.get_copy("sample7.dss") + + with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with self.assertRaises(PermissionError): + dss.delete(PATHNAME) + + def test_write_precompressed_grid_on_readonly_raises(self): + filename = self.test_files.get_copy("sample7.dss") + + with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with self.assertRaises(PermissionError): + dss.writePrecompressedGrid(None, b"\x00", 1) + + def test_readonly_reads_do_not_modify_the_file(self): + filename = self.test_files.get_copy("sample7.dss") + before = _file_fingerprint(filename) + + with HecDss(filename, DssAccess.READ_ACCESS) as dss: + for _ in range(10): + dss.get(PATHNAME, START, END) + + self.assertEqual(before, _file_fingerprint(filename)) + + def test_two_readonly_handles_open_at_once(self): + """Two handles on one file within a single process.""" + filename = self.test_files.get_copy("sample7.dss") + + first = HecDss(filename, DssAccess.READ_ACCESS) + second = HecDss(filename, DssAccess.READ_ACCESS) + try: + values_first = list(first.get(PATHNAME, START, END).values) + values_second = list(second.get(PATHNAME, START, END).values) + self.assertEqual(values_first, values_second) + finally: + first.close() + second.close() + + +class TestMultiProcessRead(unittest.TestCase): + + def setUp(self) -> None: + self.test_files = FileManager() + HecDss.set_global_debug_level(0) + self.filename = self.test_files.get_copy("sample7.dss") + with HecDss(self.filename) as dss: + self.expected = list(dss.get(PATHNAME, START, END).values) + self.assertTrue(self.expected, "test fixture returned no values") + + def tearDown(self) -> None: + self.test_files.cleanup() + + def test_concurrent_processes_read_the_same_file(self): + results = _run_workers(_read_once, [self.filename] * PROCESS_COUNT) + + self.assertEqual(PROCESS_COUNT, len(results)) + for values in results: + self.assertEqual(self.expected, values) + + def test_concurrent_processes_read_repeatedly(self): + args = [ + (self.filename, READS_PER_PROCESS, self.expected) + for _ in range(PROCESS_COUNT) + ] + results = _run_workers(_read_repeatedly, args) + + pids = {pid for pid, _ in results} + self.assertEqual( + PROCESS_COUNT, + len(pids), + f"expected {PROCESS_COUNT} distinct worker processes, got {pids}", + ) + + total = sum(matches for _, matches in results) + self.assertEqual(PROCESS_COUNT * READS_PER_PROCESS, total) + + def test_concurrent_readers_leave_the_file_unchanged(self): + before = _file_fingerprint(self.filename) + + _run_workers(_read_once, [self.filename] * PROCESS_COUNT) + + self.assertEqual( + before, + _file_fingerprint(self.filename), + "concurrent read-only readers modified the file", + ) + + def test_concurrent_readers_alongside_a_reader_in_this_process(self): + """A read-only handle here stays usable while workers read the file.""" + with HecDss(self.filename, DssAccess.READ_ACCESS) as dss: + results = _run_workers(_read_once, [self.filename] * PROCESS_COUNT) + after = list(dss.get(PATHNAME, START, END).values) + + for values in results: + self.assertEqual(self.expected, values) + self.assertEqual(self.expected, after) + + +if __name__ == "__main__": + unittest.main() From 98b93fe614f97e2c5a48df7d5f46a93350e7fd40 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Mon, 10 Aug 2026 21:04:16 +0000 Subject: [PATCH 2/2] More clear variable names and formatting --- Readme.md | 10 ++++----- src/hecdss/__init__.py | 9 ++++---- src/hecdss/download_hecdss.py | 2 +- src/hecdss/dss_access.py | 6 +++--- src/hecdss/hecdss.py | 36 +++++++++++++++---------------- src/hecdss/native.py | 12 +++++------ tests/test_access.py | 40 +++++++++++++++++------------------ 7 files changed, 57 insertions(+), 58 deletions(-) diff --git a/Readme.md b/Readme.md index 3dc5ddd..08f5b69 100644 --- a/Readme.md +++ b/Readme.md @@ -15,7 +15,7 @@ pip install hecdss [API Documentation](https://hydrologicengineeringcenter.github.io/hec-dss-python/) ### DSS file methods -1. `HecDss(file_path: str, access: DssAccess = DssAccess.GENERAL_ACCESS)`: Opens a DSS file located at the provided file path. See [File Access Modes](#file-access-modes). +1. `HecDss(file_path: str, access: OpenAccess = OpenAccess.GENERAL_ACCESS)`: Opens a DSS file located at the provided file path. See [File Access Modes](#file-access-modes). 2. `get(record_path: str)`: Retrieves the record data from the currently opened DSS file of the designated path. @@ -155,7 +155,7 @@ This library uses (`hecdss.dll` on Windows and `libhecdss.so` on Unix/Linux). h `HecDss` takes an optional `access` argument that controls how the DSS file is opened. These modes map directly to the `access` argument of `hec_dss_open_ex` in the HEC-DSS C library. -| `DssAccess` | Value | Description | +| `OpenAccess` | Value | Description | | ----------- | ----- | ----------- | | `GENERAL_ACCESS` | 0 | Read, or read/write when the file allows it. No error when the file does not have write permission. This is the default. | | `READ_ACCESS` | 1 | Read only. The file must already exist and is never written to, so several processes can read the same file at the same time. | @@ -164,12 +164,12 @@ modes map directly to the `access` argument of `hec_dss_open_ex` in the HEC-DSS | `EXCLUSIVE_ACCESS` | 4 | Exclusive write, used for squeezing. Errors when exclusive access is not available. | ```python -from hecdss import DssAccess, HecDss +from hecdss import OpenAccess, HecDss # open a file read only; several processes can do this at the same time -with HecDss("example.dss", DssAccess.READ_ACCESS) as dss: +with HecDss("example.dss", OpenAccess.READ_ACCESS) as dss: data = dss.get("/example/data/////") - print(dss.access, dss.readonly) # DssAccess.READ_ACCESS True + print(dss.access, dss.readonly) # OpenAccess.READ_ACCESS True ``` A file opened with `READ_ACCESS` must already exist; opening a missing file raises `FileNotFoundError` diff --git a/src/hecdss/__init__.py b/src/hecdss/__init__.py index 225afcc..25584a2 100644 --- a/src/hecdss/__init__.py +++ b/src/hecdss/__init__.py @@ -1,11 +1,10 @@ +from hecdss.array_container import ArrayContainer from hecdss.catalog import Catalog -from hecdss.dss_access import DssAccess -from hecdss.hecdss import HecDss +from hecdss.dss_access import OpenAccess from hecdss.dsspath import DssPath +from hecdss.hecdss import HecDss from hecdss.irregular_timeseries import IrregularTimeSeries -from hecdss.regular_timeseries import RegularTimeSeries -from hecdss.array_container import ArrayContainer from hecdss.paired_data import PairedData +from hecdss.regular_timeseries import RegularTimeSeries from hecdss.text import Text - diff --git a/src/hecdss/download_hecdss.py b/src/hecdss/download_hecdss.py index d275153..e1b513f 100644 --- a/src/hecdss/download_hecdss.py +++ b/src/hecdss/download_hecdss.py @@ -22,7 +22,7 @@ def download_and_unzip(url, zip_file, destination_dir): """ print(url) os.makedirs(destination_dir, exist_ok=True) - response = requests.get(zip_url, timeout=300) + response = requests.get(url, timeout=300) if response.status_code == 200: zip_file_path = os.path.join(destination_dir, zip_file) with open(zip_file_path, "wb") as zip_file: diff --git a/src/hecdss/dss_access.py b/src/hecdss/dss_access.py index 1fc0b56..6fa64c6 100644 --- a/src/hecdss/dss_access.py +++ b/src/hecdss/dss_access.py @@ -1,14 +1,14 @@ from enum import IntEnum -class DssAccess(IntEnum): - """DssAccess is an enumeration of the ways a DSS file can be opened +class OpenAccess(IntEnum): + """OpenAccess is an enumeration of the ways a DSS file can be opened The values match the access argument of hec_dss_open_ex in the HEC-DSS C library. Returns: - DssAccess: the access mode + OpenAccess: the access mode """ GENERAL_ACCESS = 0 diff --git a/src/hecdss/hecdss.py b/src/hecdss/hecdss.py index 0d3b981..e70b38a 100644 --- a/src/hecdss/hecdss.py +++ b/src/hecdss/hecdss.py @@ -8,7 +8,7 @@ from hecdss.array_container import ArrayContainer from hecdss.catalog import Catalog from hecdss.dateconverter import DateConverter -from hecdss.dss_access import DssAccess +from hecdss.dss_access import OpenAccess from hecdss.dsspath import DssPath from hecdss.gridded_data import GriddedData from hecdss.irregular_timeseries import IrregularTimeSeries @@ -28,20 +28,20 @@ class HecDss: """ Main class for working with DSS files """ - def __init__(self, filename: str, access: DssAccess = DssAccess.GENERAL_ACCESS): + def __init__(self, filename: str, access: OpenAccess = OpenAccess.GENERAL_ACCESS): """constructor for HecDSS Args: filename (str): DSS filename to be opened; it will be created if it - doesn't exist, unless access is DssAccess.READ_ACCESS. - access (DssAccess): how the file is opened. dss_access.py defines access modes + doesn't exist, unless access is OpenAccess.READ_ACCESS. + access (OpenAccess): how the file is opened. dss_access.py defines access modes Raises: - ValueError: access is not one of the DssAccess values. + ValueError: access is not one of the OpenAccess values. FileNotFoundError: access is READ_ACCESS and the file does not exist. """ - self._access = DssAccess(access) + self._access = OpenAccess(access) self._native = _Native() self._native.hec_dss_open(filename, self._access) self._catalog = None @@ -49,23 +49,23 @@ def __init__(self, filename: str, access: DssAccess = DssAccess.GENERAL_ACCESS): self._closed = False @property - def access(self) -> DssAccess: - """DssAccess: the access mode this file was opened with""" + def access(self) -> OpenAccess: + """OpenAccess: the access mode this file was opened with""" return self._access @property def readonly(self) -> bool: - """bool: True when this file was opened with DssAccess.READ_ACCESS""" - return self._access == DssAccess.READ_ACCESS + """bool: True when this file was opened with OpenAccess.READ_ACCESS""" + return self._access == OpenAccess.READ_ACCESS - def _check_writable(self, operation: str) -> None: + def _assert_writable(self, operation: str) -> None: """raises if this file was opened read only Args: operation (str): name of the method being attempted, used in the message. Raises: - PermissionError: the file was opened with DssAccess.READ_ACCESS. + PermissionError: the file was opened with OpenAccess.READ_ACCESS. """ if self.readonly: raise PermissionError( @@ -643,12 +643,12 @@ def put(self, container) -> int: Raises: NotImplementedError: if saving the type of container is not supported. - PermissionError: if the file was opened with DssAccess.READ_ACCESS. + PermissionError: if the file was opened with OpenAccess.READ_ACCESS. Returns: int: status of zero when successful. Non zero for errors. """ - self._check_writable("put") + self._assert_writable("put") # TODO. is timezone needed? status = 0 if type(container) is RegularTimeSeries: @@ -749,11 +749,11 @@ def writePrecompressedGrid(self, gd, compressedData, CompressionSize): compressedData (bytes): Compressed data. CompressionSize (int): Size of the compressed data. Raises: - PermissionError: if the file was opened with DssAccess.READ_ACCESS. + PermissionError: if the file was opened with OpenAccess.READ_ACCESS. Returns: int: 0 if successful, -1 otherwise. """ - self._check_writable("writePrecompressedGrid") + self._assert_writable("writePrecompressedGrid") if compressedData and CompressionSize > 0: status = self._native.hec_dss_gridStore(gd, compressedData, CompressionSize) @@ -769,11 +769,11 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en startdatetime (datetime): start date for query, if only start date is provided, delete all records at or after this date enddatetime (datetime): end date for the query, if only end date is provided, delete all records at or before this date Raises: - PermissionError: if the file was opened with DssAccess.READ_ACCESS. + PermissionError: if the file was opened with OpenAccess.READ_ACCESS. Returns: int: status of zero when successful. Non zero for errors. """ - self._check_writable("delete") + self._assert_writable("delete") rt = self.get_record_type(pathname) delete_path = DssPath(pathname) if (rt == RecordType.RegularTimeSeries or rt == RecordType.IrregularTimeSeries) and allrecords: diff --git a/src/hecdss/native.py b/src/hecdss/native.py index af5a1c7..afcb5bd 100644 --- a/src/hecdss/native.py +++ b/src/hecdss/native.py @@ -19,7 +19,7 @@ import numpy as np -from hecdss.dss_access import DssAccess +from hecdss.dss_access import OpenAccess # from hecdss.location_info import LocationInfo @@ -78,13 +78,13 @@ def __init__(self): else: self.dll = self.load_hecdss_library("libhecdss.so") - def hec_dss_open(self, dss_filename: str, access: int = DssAccess.GENERAL_ACCESS) -> int: + def hec_dss_open(self, dss_filename: str, access: int = OpenAccess.GENERAL_ACCESS) -> int: """opens a DSS file and gets a handle Args: dss_filename (str): filename to open; it is created if it doesn't exist, except when access is READ_ACCESS. - access (int): read/write access used to open the file, see DssAccess + access (int): read/write access used to open the file, see OpenAccess 0 - GENERAL_ACCESS: Doesn't matter (no error if file doesn't have write permission) 1 - READ_ACCESS: Read only (will not allow writing to file) 2 - MULTI_USER_ACCESS: Read/Write permission with full multi-user access @@ -98,13 +98,13 @@ def hec_dss_open(self, dss_filename: str, access: int = DssAccess.GENERAL_ACCESS int: status of zero when successful, non-zero on error. Raises: - ValueError: access is not one of the DssAccess values. + ValueError: access is not one of the OpenAccess values. FileNotFoundError: READ_ACCESS was requested and the file is missing. Exception: the file could not be opened. """ - access = DssAccess(access) + access = OpenAccess(access) - if access == DssAccess.READ_ACCESS and not os.path.exists(dss_filename): + if access == OpenAccess.READ_ACCESS and not os.path.exists(dss_filename): raise FileNotFoundError( f"DSS file not found: '{dss_filename}'. " f"A file must already exist to be opened with {access.name}." diff --git a/tests/test_access.py b/tests/test_access.py index 74d9b06..3ce7c6d 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -7,7 +7,7 @@ from file_manager import FileManager -from hecdss import DssAccess, HecDss +from hecdss import HecDss, OpenAccess PATHNAME = "//SACRAMENTO/PRECIP-INC//1Day/OBS/" @@ -15,10 +15,10 @@ END = datetime(2005, 1, 4) READ_WRITE_ACCESS = [ - DssAccess.GENERAL_ACCESS, - DssAccess.MULTI_USER_ACCESS, - DssAccess.SINGLE_USER_ADVISORY_ACCESS, - DssAccess.EXCLUSIVE_ACCESS, + OpenAccess.GENERAL_ACCESS, + OpenAccess.MULTI_USER_ACCESS, + OpenAccess.SINGLE_USER_ADVISORY_ACCESS, + OpenAccess.EXCLUSIVE_ACCESS, ] PROCESS_COUNT = 4 @@ -36,7 +36,7 @@ def _file_fingerprint(filename): def _read_once(filename): """Opens filename read-only and returns the values of PATHNAME.""" HecDss.set_global_debug_level(0) - with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with HecDss(filename, OpenAccess.READ_ACCESS) as dss: return list(dss.get(PATHNAME, START, END).values) @@ -48,7 +48,7 @@ def _read_repeatedly(args): filename, iterations, expected = args HecDss.set_global_debug_level(0) matches = 0 - with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with HecDss(filename, OpenAccess.READ_ACCESS) as dss: for _ in range(iterations): if list(dss.get(PATHNAME, START, END).values) == expected: matches += 1 @@ -79,7 +79,7 @@ def test_every_access_mode_reads_the_same_data(self): expected = list(dss.get(PATHNAME, START, END).values) self.assertTrue(expected, "test fixture returned no values") - for access in DssAccess: + for access in OpenAccess: with self.subTest(access=access.name): # a fresh copy per mode, so a writable mode cannot affect the next filename = self.test_files.get_copy("sample7.dss") @@ -90,14 +90,14 @@ def test_default_access_is_general_access(self): filename = self.test_files.get_copy("sample7.dss") with HecDss(filename) as dss: - self.assertEqual(DssAccess.GENERAL_ACCESS, dss.access) + self.assertEqual(OpenAccess.GENERAL_ACCESS, dss.access) self.assertFalse(dss.readonly) def test_access_is_reported_and_plain_ints_are_accepted(self): filename = self.test_files.get_copy("sample7.dss") with HecDss(filename, 1) as dss: - self.assertEqual(DssAccess.READ_ACCESS, dss.access) + self.assertEqual(OpenAccess.READ_ACCESS, dss.access) self.assertTrue(dss.readonly) def test_invalid_access_raises_value_error(self): @@ -120,7 +120,7 @@ def test_read_write_modes_can_write(self): ts.values = ts.values * 2 self.assertEqual(0, dss.put(ts)) - with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with HecDss(filename, OpenAccess.READ_ACCESS) as dss: self.assertEqual(expected, list(dss.get(PATHNAME, START, END).values)) @@ -141,7 +141,7 @@ def test_readonly_catalog_and_record_count(self): with HecDss(filename) as dss: expected_count = dss.record_count() - with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with HecDss(filename, OpenAccess.READ_ACCESS) as dss: self.assertEqual(expected_count, dss.record_count()) self.assertTrue(len(dss.get_catalog().uncondensed_paths) > 0) @@ -150,7 +150,7 @@ def test_readonly_missing_file_raises_and_does_not_create_it(self): self.assertFalse(os.path.exists(missing)) with self.assertRaises(FileNotFoundError): - HecDss(missing, DssAccess.READ_ACCESS) + HecDss(missing, OpenAccess.READ_ACCESS) self.assertFalse( os.path.exists(missing), @@ -172,21 +172,21 @@ def test_put_on_readonly_raises(self): with HecDss(filename) as dss: container = dss.get(PATHNAME, START, END) - with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with HecDss(filename, OpenAccess.READ_ACCESS) as dss: with self.assertRaises(PermissionError): dss.put(container) def test_delete_on_readonly_raises(self): filename = self.test_files.get_copy("sample7.dss") - with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with HecDss(filename, OpenAccess.READ_ACCESS) as dss: with self.assertRaises(PermissionError): dss.delete(PATHNAME) def test_write_precompressed_grid_on_readonly_raises(self): filename = self.test_files.get_copy("sample7.dss") - with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with HecDss(filename, OpenAccess.READ_ACCESS) as dss: with self.assertRaises(PermissionError): dss.writePrecompressedGrid(None, b"\x00", 1) @@ -194,7 +194,7 @@ def test_readonly_reads_do_not_modify_the_file(self): filename = self.test_files.get_copy("sample7.dss") before = _file_fingerprint(filename) - with HecDss(filename, DssAccess.READ_ACCESS) as dss: + with HecDss(filename, OpenAccess.READ_ACCESS) as dss: for _ in range(10): dss.get(PATHNAME, START, END) @@ -204,8 +204,8 @@ def test_two_readonly_handles_open_at_once(self): """Two handles on one file within a single process.""" filename = self.test_files.get_copy("sample7.dss") - first = HecDss(filename, DssAccess.READ_ACCESS) - second = HecDss(filename, DssAccess.READ_ACCESS) + first = HecDss(filename, OpenAccess.READ_ACCESS) + second = HecDss(filename, OpenAccess.READ_ACCESS) try: values_first = list(first.get(PATHNAME, START, END).values) values_second = list(second.get(PATHNAME, START, END).values) @@ -265,7 +265,7 @@ def test_concurrent_readers_leave_the_file_unchanged(self): def test_concurrent_readers_alongside_a_reader_in_this_process(self): """A read-only handle here stays usable while workers read the file.""" - with HecDss(self.filename, DssAccess.READ_ACCESS) as dss: + with HecDss(self.filename, OpenAccess.READ_ACCESS) as dss: results = _run_workers(_read_once, [self.filename] * PROCESS_COUNT) after = list(dss.get(PATHNAME, START, END).values)