merge: PR #159 from dev

Weekly release 2024-03-11

ckanext/versioned_datastore/lib/query/schema.py

@@ -98,6 +98,20 @@ def hash_query(query, version):
         return schemas[version].hash(query)
 
 
+def normalise_query(query, version):
+    """
+    Corrects some (small) common query errors, e.g. removing empty groups.
+
+    :param query: the query dict
+    :param version: the query version
+    :return: the corrected/normalised query
+    """
+    if version not in schemas:
+        raise InvalidQuerySchemaVersionError(version)
+    else:
+        return schemas[version].normalise(query)
+
+
 def load_core_schema(version):
     """
     Given a query schema version, loads the schema from the schema_base_path directory.
@@ -155,3 +169,12 @@ def hash(self, query):
         :return: a string hex digest
         """
         pass
+
+    def normalise(self, query):
+        """
+        Corrects some (small) common query errors, e.g. removing empty groups.
+
+        :param query: the query dict
+        :return: the corrected/normalised query dict
+        """
+        return query

ckanext/versioned_datastore/lib/query/utils.py

@@ -296,3 +296,60 @@ def get_resources_and_versions(
             rounded_resource_ids_and_versions[resource_id] = rounded_version
 
     return available_resource_ids, rounded_resource_ids_and_versions
+
+
+def convert_small_or_groups(query):
+    """
+    Convert OR groups containing only 1 item to AND groups.
+
+    :param query: a multisearch query dict
+    :return: the query with a converted filter dict, if applicable
+    """
+    if 'filters' not in query:
+        return query
+
+    def _convert(*filters):
+        items = []
+        for term_or_group in filters:
+            k, v = list(term_or_group.items())[0]
+            if k not in ['and', 'or', 'not']:
+                items.append(term_or_group)
+            elif k != 'or' or len(v) != 1:
+                # don't convert empty groups because those throw an error for all types
+                items.append({k: _convert(*v)})
+            else:
+                items.append({'and': _convert(*v)})
+        return items
+
+    query['filters'] = _convert(query['filters'])[0]
+
+    return query
+
+
+def remove_empty_groups(query):
+    """
+    Remove empty groups from filter list.
+
+    :param query: a multisearch query dict
+    :return: the query with a processed filter dict, if applicable
+    """
+    if 'filters' not in query:
+        return query
+
+    def _convert(*filters):
+        items = []
+        for term_or_group in filters:
+            k, v = list(term_or_group.items())[0]
+            if k not in ['and', 'or', 'not']:
+                items.append(term_or_group)
+            elif len(v) > 0:
+                items.append({k: _convert(*v)})
+        return items
+
+    processed_filters = _convert(query['filters'])
+    if len(processed_filters) == 0:
+        del query['filters']
+    else:
+        query['filters'] = processed_filters[0]
+
+    return query

ckanext/versioned_datastore/lib/query/v1_0_0.py

@@ -9,6 +9,7 @@
 
 from .schema import Schema, load_core_schema, schema_base_path
 from ..datastore_utils import prefix_field
+from .utils import convert_small_or_groups, remove_empty_groups
 
 
 class v1_0_0Schema(Schema):
@@ -64,6 +65,17 @@ def translate(self, query, search=None):
         search = self.add_filters(query, search)
         return search
 
+    def normalise(self, query):
+        """
+        Corrects some (small) common query errors, e.g. removing empty groups.
+
+        :param query: the query dict
+        :return: the corrected/normalised query dict
+        """
+        query = convert_small_or_groups(query)
+        query = remove_empty_groups(query)
+        return query
+
     def add_search(self, query, search):
         """
         Adds a search to the search object and then returns it. Search terms map

ckanext/versioned_datastore/logic/actions/multisearch.py

@@ -33,6 +33,7 @@
     validate_query,
     translate_query,
     hash_query,
+    normalise_query,
 )
 from ...lib.query.slugs import create_slug, resolve_slug, create_nav_slug
 from ...lib.query.utils import (
@@ -96,6 +97,8 @@ def datastore_multisearch(
 
     timer = Timer()
 
+    query = normalise_query(query, query_version)
+
     try:
         # validate and translate the query into an elasticsearch-dsl Search object
         validate_query(query, query_version)
@@ -126,7 +129,7 @@ def datastore_multisearch(
     # of the modified date, id of the record and the index it's in so that we get a unique sort
     search = search.sort(
         # not all indexes have a modified field so we need to provide the unmapped_type option
-        {'data.modified': {'order': 'desc', 'unmapped_type': 'date'}},
+        {'meta.version': 'desc'},
         {'data._id': 'desc'},
         {'_index': 'desc'},
     )
@@ -424,6 +427,8 @@ def datastore_guess_fields(
         set(g.lower() for g in ignore_groups) if ignore_groups is not None else set()
     )
 
+    query = normalise_query(query, query_version)
+
     try:
         # validate and translate the query into an elasticsearch-dsl Search object
         validate_query(query, query_version)
@@ -520,6 +525,8 @@ def datastore_value_autocomplete(
     # limit the size so that it is between 1 and 500
     size = max(1, min(size, 500))
 
+    query = normalise_query(query, query_version)
+
     try:
         # validate and translate the query into an elasticsearch-dsl Search object
         validate_query(query, query_version)
@@ -601,6 +608,8 @@ def datastore_hash_query(query=None, query_version=None):
     if query_version is None:
         query_version = get_latest_query_version()
 
+    query = normalise_query(query, query_version)
+
     try:
         validate_query(query, query_version)
     except (jsonschema.ValidationError, InvalidQuerySchemaVersionError) as e:
@@ -668,6 +677,8 @@ def datastore_multisearch_counts(
     if query_version is None:
         query_version = get_latest_query_version()
 
+    query = normalise_query(query, query_version)
+
     try:
         # validate and translate the query into an elasticsearch-dsl Search object
         validate_query(query, query_version)

tests/unit/lib/query/test_query_utils.py

@@ -0,0 +1,54 @@
+from ckanext.versioned_datastore.lib.query.utils import (
+    convert_small_or_groups,
+    remove_empty_groups,
+)
+
+
+class TestQueryUtils(object):
+    def test_converts_single_item_or_group(self):
+        test_query = {
+            'filters': {'or': [{'string_equals': {'fields': ['x'], 'value': 'y'}}]}
+        }
+        expected_query = {
+            'filters': {'and': [{'string_equals': {'fields': ['x'], 'value': 'y'}}]}
+        }
+        output_query = convert_small_or_groups(test_query)
+        assert output_query == expected_query
+
+    def test_converts_nested_single_item_or_group(self):
+        test_query = {'filters': {'and': [{'or': [{'exists': {'fields': ['x']}}]}]}}
+        expected_query = {
+            'filters': {'and': [{'and': [{'exists': {'fields': ['x']}}]}]}
+        }
+        output_query = convert_small_or_groups(test_query)
+        assert output_query == expected_query
+
+    def test_does_not_convert_multi_item_or_group(self):
+        test_query = {
+            'filters': {
+                'or': [{'exists': {'fields': ['x']}}, {'exists': {'fields': ['y']}}]
+            }
+        }
+        output_query = convert_small_or_groups(test_query)
+        assert output_query == test_query
+
+    def test_does_not_convert_empty_or_group(self):
+        test_query = {'filters': {'or': []}}
+        output_query = convert_small_or_groups(test_query)
+        assert output_query == test_query
+
+    def test_removes_empty_groups(self):
+        test_query = {
+            'filters': {'and': [{'or': []}, {'not': [{'exists': {'fields': ['x']}}]}]}
+        }
+        expected_query = {
+            'filters': {'and': [{'not': [{'exists': {'fields': ['x']}}]}]}
+        }
+        output_query = remove_empty_groups(test_query)
+        assert output_query == expected_query
+
+    def test_removes_filters_if_root_empty(self):
+        test_query = {'filters': {'and': []}}
+        expected_query = {}
+        output_query = remove_empty_groups(test_query)
+        assert output_query == expected_query