fix: http head sometimes return 405

src/neuroglancer_scripts/http_accessor.py

@@ -59,6 +59,10 @@ def file_exists(self, relative_path):
             r = self._session.head(file_url)
             if r.status_code == requests.codes.not_found:
                 return False
+            if r.status_code == 405:
+                r = self._session.get(file_url, stream=True)
+                r.raise_for_status()
+                return True
             r.raise_for_status()
         except requests.exceptions.RequestException as exc:
             raise DataAccessError("Error probing the existence of "

src/neuroglancer_scripts/sharded_http_accessor.py

@@ -40,9 +40,13 @@ def __init__(self, base_url, session: requests.Session,
         assert self.can_read_cmc
 
     def file_exists(self, filepath):
-        resp = self._session.head(f"{self.base_url}{filepath}")
+        url = f"{self.base_url}{filepath}"
+        resp = self._session.head(url)
         if resp.status_code in (200, 404):
             return resp.status_code == 200
+        if resp.status_code == 405:
+            resp = self._session.get(url, stream=True)
+            return resp.status_code == 200
         resp.raise_for_status()
         return False
 
@@ -74,8 +78,10 @@ def read_bytes(self, offset: int, length: int) -> bytes:
 
     def fetch_cmc_chunk(self, cmc: np.uint64):
         minishard_key = self.get_minishard_key(cmc)
-        assert minishard_key in self.minishard_dict
-        return self.minishard_dict[minishard_key].fetch_cmc_chunk(cmc)
+        assert (
+            minishard_key in self.ro_minishard_dict
+        ), f"{minishard_key} not in {self.minishard_dict.keys()}"
+        return self.ro_minishard_dict[minishard_key].fetch_cmc_chunk(cmc)
 
 
 class HttpShardedScale(ShardedScaleBase):

unit_tests/test_http_accessor.py

@@ -33,6 +33,10 @@ def test_http_accessor(base_url, requests_mock):
     requests_mock.head("http://h.test/i/info", status_code=404)
     assert a.file_exists("info") is False
 
+    requests_mock.head("http://h.test/i/info", status_code=405)
+    requests_mock.get("http://h.test/i/info", status_code=200)
+    assert a.file_exists("info") is True
+
     requests_mock.get("http://h.test/i/key/0-1_0-1_0-1",
                       content=dummy_chunk_buf)
     fetched_chunk = a.fetch_chunk("key", chunk_coords)
@@ -47,6 +51,11 @@ def test_http_accessor_errors(requests_mock):
     with pytest.raises(DataAccessError):
         a.file_exists("info")
 
+    requests_mock.head("http://h.test/i/info", status_code=405)
+    requests_mock.get("http://h.test/i/info", status_code=404)
+    with pytest.raises(DataAccessError):
+        a.file_exists("info")
+
     requests_mock.get("http://h.test/i/info", status_code=404)
     with pytest.raises(DataAccessError):
         a.fetch_file("info")