Skip to content

Commit bcee30c

Browse files
rhysrevans3jonhealy1
rhysrevans3
and
jonhealy1
authored
Dynamic queryables mapping for properties (#375)
**Description:** Separating queryables mapping from #340 this allows users to search using the property name without needing to prepend `properties.` **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --------- Co-authored-by: Jonathan Healy <jonathan.d.healy@gmail.com>
1 parent bc1c3f7 commit bcee30c

File tree

7 files changed

+84
-32
lines changed

7 files changed

+84
-32
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
99

1010
### Added
1111

12+
- Added dynamic queryables mapping for search and aggregations [#375](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/375)
1213
- Added configurable landing page ID `STAC_FASTAPI_LANDING_PAGE_ID` [#352](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/352)
1314
- Added support for `S_CONTAINS`, `S_WITHIN`, `S_DISJOINT` spatial filter operations [#371](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/371)
1415
- Introduced the `DATABASE_REFRESH` environment variable to control whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. [#370](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/370)

‎stac_fastapi/core/stac_fastapi/core/core.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -607,7 +607,7 @@ async def post_search(
607607
if hasattr(search_request, "filter_expr"):
608608
cql2_filter = getattr(search_request, "filter_expr", None)
609609
try:
610-
search = self.database.apply_cql2_filter(search, cql2_filter)
610+
search = await self.database.apply_cql2_filter(search, cql2_filter)
611611
except Exception as e:
612612
raise HTTPException(
613613
status_code=400, detail=f"Error with cql2_json filter: {e}"

‎stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,7 @@ async def aggregate(
467467

468468
if aggregate_request.filter_expr:
469469
try:
470-
search = self.database.apply_cql2_filter(
470+
search = await self.database.apply_cql2_filter(
471471
search, aggregate_request.filter_expr
472472
)
473473
except Exception as e:

‎stac_fastapi/core/stac_fastapi/core/extensions/filter.py

+15-22
Original file line numberDiff line numberDiff line change
@@ -91,20 +91,7 @@ class SpatialOp(str, Enum):
9191
S_DISJOINT = "s_disjoint"
9292

9393

94-
queryables_mapping = {
95-
"id": "id",
96-
"collection": "collection",
97-
"geometry": "geometry",
98-
"datetime": "properties.datetime",
99-
"created": "properties.created",
100-
"updated": "properties.updated",
101-
"cloud_cover": "properties.eo:cloud_cover",
102-
"cloud_shadow_percentage": "properties.s2:cloud_shadow_percentage",
103-
"nodata_pixel_percentage": "properties.s2:nodata_pixel_percentage",
104-
}
105-
106-
107-
def to_es_field(field: str) -> str:
94+
def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
10895
"""
10996
Map a given field to its corresponding Elasticsearch field according to a predefined mapping.
11097
@@ -117,7 +104,7 @@ def to_es_field(field: str) -> str:
117104
return queryables_mapping.get(field, field)
118105

119106

120-
def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
107+
def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str, Any]:
121108
"""
122109
Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
123110
@@ -133,7 +120,13 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
133120
LogicalOp.OR: "should",
134121
LogicalOp.NOT: "must_not",
135122
}[query["op"]]
136-
return {"bool": {bool_type: [to_es(sub_query) for sub_query in query["args"]]}}
123+
return {
124+
"bool": {
125+
bool_type: [
126+
to_es(queryables_mapping, sub_query) for sub_query in query["args"]
127+
]
128+
}
129+
}
137130

138131
elif query["op"] in [
139132
ComparisonOp.EQ,
@@ -150,7 +143,7 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
150143
ComparisonOp.GTE: "gte",
151144
}
152145

153-
field = to_es_field(query["args"][0]["property"])
146+
field = to_es_field(queryables_mapping, query["args"][0]["property"])
154147
value = query["args"][1]
155148
if isinstance(value, dict) and "timestamp" in value:
156149
value = value["timestamp"]
@@ -173,11 +166,11 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
173166
return {"range": {field: {range_op[query["op"]]: value}}}
174167

175168
elif query["op"] == ComparisonOp.IS_NULL:
176-
field = to_es_field(query["args"][0]["property"])
169+
field = to_es_field(queryables_mapping, query["args"][0]["property"])
177170
return {"bool": {"must_not": {"exists": {"field": field}}}}
178171

179172
elif query["op"] == AdvancedComparisonOp.BETWEEN:
180-
field = to_es_field(query["args"][0]["property"])
173+
field = to_es_field(queryables_mapping, query["args"][0]["property"])
181174
gte, lte = query["args"][1], query["args"][2]
182175
if isinstance(gte, dict) and "timestamp" in gte:
183176
gte = gte["timestamp"]
@@ -186,14 +179,14 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
186179
return {"range": {field: {"gte": gte, "lte": lte}}}
187180

188181
elif query["op"] == AdvancedComparisonOp.IN:
189-
field = to_es_field(query["args"][0]["property"])
182+
field = to_es_field(queryables_mapping, query["args"][0]["property"])
190183
values = query["args"][1]
191184
if not isinstance(values, list):
192185
raise ValueError(f"Arg {values} is not a list")
193186
return {"terms": {field: values}}
194187

195188
elif query["op"] == AdvancedComparisonOp.LIKE:
196-
field = to_es_field(query["args"][0]["property"])
189+
field = to_es_field(queryables_mapping, query["args"][0]["property"])
197190
pattern = cql2_like_to_es(query["args"][1])
198191
return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
199192

@@ -203,7 +196,7 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
203196
SpatialOp.S_WITHIN,
204197
SpatialOp.S_DISJOINT,
205198
]:
206-
field = to_es_field(query["args"][0]["property"])
199+
field = to_es_field(queryables_mapping, query["args"][0]["property"])
207200
geometry = query["args"][1]
208201

209202
relation_mapping = {

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py

+32-3
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,34 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict:
290290
)
291291
return item["_source"]
292292

293+
async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
294+
"""Retrieve mapping of Queryables for search.
295+
296+
Args:
297+
collection_id (str, optional): The id of the Collection the Queryables
298+
belongs to. Defaults to "*".
299+
300+
Returns:
301+
dict: A dictionary containing the Queryables mappings.
302+
"""
303+
queryables_mapping = {}
304+
305+
mappings = await self.client.indices.get_mapping(
306+
index=f"{ITEMS_INDEX_PREFIX}{collection_id}",
307+
)
308+
309+
for mapping in mappings.values():
310+
fields = mapping["mappings"].get("properties", {})
311+
properties = fields.pop("properties", {}).get("properties", {}).keys()
312+
313+
for field_key in fields:
314+
queryables_mapping[field_key] = field_key
315+
316+
for property_key in properties:
317+
queryables_mapping[property_key] = f"properties.{property_key}"
318+
319+
return queryables_mapping
320+
293321
@staticmethod
294322
def make_search():
295323
"""Database logic to create a Search instance."""
@@ -518,8 +546,9 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]
518546

519547
return search
520548

521-
@staticmethod
522-
def apply_cql2_filter(search: Search, _filter: Optional[Dict[str, Any]]):
549+
async def apply_cql2_filter(
550+
self, search: Search, _filter: Optional[Dict[str, Any]]
551+
):
523552
"""
524553
Apply a CQL2 filter to an Elasticsearch Search object.
525554
@@ -539,7 +568,7 @@ def apply_cql2_filter(search: Search, _filter: Optional[Dict[str, Any]]):
539568
otherwise the original Search object.
540569
"""
541570
if _filter is not None:
542-
es_query = filter.to_es(_filter)
571+
es_query = filter.to_es(await self.get_queryables_mapping(), _filter)
543572
search = search.query(es_query)
544573

545574
return search

‎stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

+32-3
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,34 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict:
307307
)
308308
return item["_source"]
309309

310+
async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
311+
"""Retrieve mapping of Queryables for search.
312+
313+
Args:
314+
collection_id (str, optional): The id of the Collection the Queryables
315+
belongs to. Defaults to "*".
316+
317+
Returns:
318+
dict: A dictionary containing the Queryables mappings.
319+
"""
320+
queryables_mapping = {}
321+
322+
mappings = await self.client.indices.get_mapping(
323+
index=f"{ITEMS_INDEX_PREFIX}{collection_id}",
324+
)
325+
326+
for mapping in mappings.values():
327+
fields = mapping["mappings"].get("properties", {})
328+
properties = fields.pop("properties", {}).get("properties", {}).keys()
329+
330+
for field_key in fields:
331+
queryables_mapping[field_key] = field_key
332+
333+
for property_key in properties:
334+
queryables_mapping[property_key] = f"properties.{property_key}"
335+
336+
return queryables_mapping
337+
310338
@staticmethod
311339
def make_search():
312340
"""Database logic to create a Search instance."""
@@ -535,8 +563,9 @@ def apply_stacql_filter(search: Search, op: str, field: str, value: float):
535563

536564
return search
537565

538-
@staticmethod
539-
def apply_cql2_filter(search: Search, _filter: Optional[Dict[str, Any]]):
566+
async def apply_cql2_filter(
567+
self, search: Search, _filter: Optional[Dict[str, Any]]
568+
):
540569
"""
541570
Apply a CQL2 filter to an Opensearch Search object.
542571
@@ -556,7 +585,7 @@ def apply_cql2_filter(search: Search, _filter: Optional[Dict[str, Any]]):
556585
otherwise the original Search object.
557586
"""
558587
if _filter is not None:
559-
es_query = filter.to_es(_filter)
588+
es_query = filter.to_es(await self.get_queryables_mapping(), _filter)
560589
search = search.filter(es_query)
561590

562591
return search

‎stac_fastapi/tests/extensions/test_filter.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ async def test_search_filter_ext_and_get_cql2text_id(app_client, ctx):
163163
async def test_search_filter_ext_and_get_cql2text_cloud_cover(app_client, ctx):
164164
collection = ctx.item["collection"]
165165
cloud_cover = ctx.item["properties"]["eo:cloud_cover"]
166-
filter = f"cloud_cover={cloud_cover} AND collection='{collection}'"
166+
filter = f"eo:cloud_cover={cloud_cover} AND collection='{collection}'"
167167
resp = await app_client.get(f"/search?filter-lang=cql2-text&filter={filter}")
168168

169169
assert resp.status_code == 200
@@ -176,7 +176,7 @@ async def test_search_filter_ext_and_get_cql2text_cloud_cover_no_results(
176176
):
177177
collection = ctx.item["collection"]
178178
cloud_cover = ctx.item["properties"]["eo:cloud_cover"] + 1
179-
filter = f"cloud_cover={cloud_cover} AND collection='{collection}'"
179+
filter = f"eo:cloud_cover={cloud_cover} AND collection='{collection}'"
180180
resp = await app_client.get(f"/search?filter-lang=cql2-text&filter={filter}")
181181

182182
assert resp.status_code == 200

0 commit comments

Comments
 (0)