diff --git a/src/datasette_reconcile/__about__.py b/src/datasette_reconcile/__about__.py index 5a55e75..6e6b3b5 100644 --- a/src/datasette_reconcile/__about__.py +++ b/src/datasette_reconcile/__about__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2023-present David Kane # # SPDX-License-Identifier: MIT -__version__ = "0.6.2" +__version__ = "0.6.3" diff --git a/src/datasette_reconcile/reconcile.py b/src/datasette_reconcile/reconcile.py index 4d1a943..098a91a 100644 --- a/src/datasette_reconcile/reconcile.py +++ b/src/datasette_reconcile/reconcile.py @@ -251,13 +251,16 @@ def _get_query_result(self, row, query): } ] - return { + result = { "id": str(row[self.config["id_field"]]), "name": name, "type": type_, "score": fuzz.ratio(name_match, query_match), "match": name_match == query_match, } + if self.config["description_field"]: + result["description"] = str(row[self.config["description_field"]]) + return result async def _service_manifest(self, request): # @todo: if type_field is set then get a list of types to use in the "defaultTypes" item below. diff --git a/src/datasette_reconcile/utils.py b/src/datasette_reconcile/utils.py index e4be586..c9ac16c 100644 --- a/src/datasette_reconcile/utils.py +++ b/src/datasette_reconcile/utils.py @@ -61,6 +61,8 @@ async def check_config(config, db, table): if "name_field" not in config: msg = "Name field must be defined to activate reconciliation" raise ReconcileError(msg) + if "description_field" not in config: + config["description_field"] = None if "type_field" not in config and "type_default" not in config: config["type_default"] = [DEFAULT_TYPE] @@ -87,7 +89,8 @@ async def check_config(config, db, table): msg = "View URL must contain {{id}}" raise ReconcileError(msg) - config["fts_table"] = await db.fts_table(table) + if "fts_table" not in config: + config["fts_table"] = await db.fts_table(table) # let's show a warning if sqlite3 version is less than 3.30.0 # full text search results will fail for < 3.30.0 if the table @@ -111,6 +114,8 @@ def get_select_fields(config): select_fields = [config["id_field"], config["name_field"], *config.get("additional_fields", [])] if config.get("type_field"): select_fields.append(config["type_field"]) + if config.get("description_field"): + select_fields.append(config["description_field"]) return select_fields diff --git a/tests/conftest.py b/tests/conftest.py index 26f1af2..241b293 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,7 +16,7 @@ ) -def create_db(tmp_path_factory): +def create_db(tmp_path_factory, enable_fts): db_directory = tmp_path_factory.mktemp("dbs") db_path = db_directory / "test.db" db = sqlite_utils.Database(db_path) @@ -29,6 +29,10 @@ def create_db(tmp_path_factory): ], pk="id", ) + + if enable_fts: + db["dogs"].enable_fts(["name"]) + return db_path @@ -55,13 +59,24 @@ def get_schema(filename): @pytest.fixture(scope="session") def ds(tmp_path_factory): - ds = Datasette([create_db(tmp_path_factory)], metadata=plugin_metadata()) + ds = Datasette([create_db(tmp_path_factory, False)], metadata=plugin_metadata()) return ds @pytest.fixture(scope="session") def db_path(tmp_path_factory): - return create_db(tmp_path_factory) + return create_db(tmp_path_factory, False) + + +@pytest.fixture(scope="session") +def ds_fts(tmp_path_factory): + ds = Datasette([create_db(tmp_path_factory, True)], metadata=plugin_metadata()) + return ds + + +@pytest.fixture(scope="session") +def db_path_fts(tmp_path_factory): + return create_db(tmp_path_factory, True) def retrieve_schema_from_filesystem(uri: str): @@ -79,3 +94,10 @@ def retrieve_schema_from_filesystem(uri: str): registry = Registry(retrieve=retrieve_schema_from_filesystem) + + +def do_method(client, method, *args, **kwargs): + if method == "post": + return client.post(*args, **kwargs) + kwargs["params"] = kwargs.pop("data") + return client.get(*args, **kwargs) diff --git a/tests/test_reconcile.py b/tests/test_reconcile.py index 1421f64..f97e9aa 100644 --- a/tests/test_reconcile.py +++ b/tests/test_reconcile.py @@ -4,7 +4,7 @@ import pytest from datasette.app import Datasette -from tests.conftest import plugin_metadata +from tests.conftest import do_method, plugin_metadata @pytest.mark.asyncio @@ -123,10 +123,13 @@ async def test_servce_manifest_view_suggest(db_path, suggest_type): @pytest.mark.asyncio -async def test_response_queries_post(db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries(db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() async with httpx.AsyncClient(app=app) as client: - response = await client.post( + response = await do_method( + client, + method, "http://localhost/test/dogs/-/reconcile", data={"queries": json.dumps({"q0": {"query": "fido"}})}, ) @@ -144,37 +147,18 @@ async def test_response_queries_post(db_path): "id": "object", } ] + assert "description" not in result assert response.headers["Access-Control-Allow-Origin"] == "*" @pytest.mark.asyncio -async def test_response_queries_get(db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_no_results(db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() async with httpx.AsyncClient(app=app) as client: - queries = json.dumps({"q0": {"query": "fido"}}) - response = await client.get(f"http://localhost/test/dogs/-/reconcile?queries={queries}") - assert 200 == response.status_code - data = response.json() - assert "q0" in data.keys() - assert len(data["q0"]["result"]) == 1 - result = data["q0"]["result"][0] - assert result["id"] == "3" - assert result["name"] == "Fido" - assert result["score"] == 100 - assert result["type"] == [ - { - "name": "Object", - "id": "object", - } - ] - assert response.headers["Access-Control-Allow-Origin"] == "*" - - -@pytest.mark.asyncio -async def test_response_queries_no_results_post(db_path): - app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() - async with httpx.AsyncClient(app=app) as client: - response = await client.post( + response = await do_method( + client, + method, "http://localhost/test/dogs/-/reconcile", data={"queries": json.dumps({"q0": {"query": "abcdef"}})}, ) @@ -185,19 +169,6 @@ async def test_response_queries_no_results_post(db_path): assert response.headers["Access-Control-Allow-Origin"] == "*" -@pytest.mark.asyncio -async def test_response_queries_no_results_get(db_path): - app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() - async with httpx.AsyncClient(app=app) as client: - queries = json.dumps({"q0": {"query": "abcdef"}}) - response = await client.get(f"http://localhost/test/dogs/-/reconcile?queries={queries}") - assert 200 == response.status_code - data = response.json() - assert "q0" in data.keys() - assert len(data["q0"]["result"]) == 0 - assert response.headers["Access-Control-Allow-Origin"] == "*" - - @pytest.mark.asyncio async def test_response_propose_properties(db_path): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() @@ -213,11 +184,12 @@ async def test_response_propose_properties(db_path): @pytest.mark.asyncio -async def test_response_extend(db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_extend(db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() async with httpx.AsyncClient(app=app) as client: extend = {"extend": json.dumps({"ids": ["1", "2", "3", "4"], "properties": [{"id": "status"}, {"id": "age"}]})} - response = await client.post("http://localhost/test/dogs/-/reconcile", data=extend) + response = await do_method(client, method, "http://localhost/test/dogs/-/reconcile", data=extend) assert 200 == response.status_code data = response.json() @@ -343,10 +315,13 @@ async def test_response_suggest_type_1(db_path): @pytest.mark.asyncio -async def test_response_queries_post_type(db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_post_type(db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "type_field": "status"})).app() async with httpx.AsyncClient(app=app) as client: - response = await client.post( + response = await do_method( + client, + method, "http://localhost/test/dogs/-/reconcile", data={"queries": json.dumps({"q0": {"query": "fido", "type": "bad dog"}})}, ) @@ -368,10 +343,13 @@ async def test_response_queries_post_type(db_path): @pytest.mark.asyncio -async def test_response_queries_post_type_list(db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_post_type_list(db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "type_field": "status"})).app() async with httpx.AsyncClient(app=app) as client: - response = await client.post( + response = await do_method( + client, + method, "http://localhost/test/dogs/-/reconcile", data={"queries": json.dumps({"q0": {"query": "fido", "type": ["bad dog"]}})}, ) @@ -393,10 +371,13 @@ async def test_response_queries_post_type_list(db_path): @pytest.mark.asyncio -async def test_response_queries_post_type_empty(db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_post_type_empty(db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "type_field": "status"})).app() async with httpx.AsyncClient(app=app) as client: - response = await client.post( + response = await do_method( + client, + method, "http://localhost/test/dogs/-/reconcile", data={"queries": json.dumps({"q0": {"query": "fido", "type": ["good dog"]}})}, ) @@ -408,10 +389,13 @@ async def test_response_queries_post_type_empty(db_path): @pytest.mark.asyncio -async def test_response_queries_post_type_not_given(db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_post_type_not_given(db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "type_field": "status"})).app() async with httpx.AsyncClient(app=app) as client: - response = await client.post( + response = await do_method( + client, + method, "http://localhost/test/dogs/-/reconcile", data={"queries": json.dumps({"q0": {"query": "fido"}})}, ) @@ -430,3 +414,34 @@ async def test_response_queries_post_type_not_given(db_path): } ] assert response.headers["Access-Control-Allow-Origin"] == "*" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_description_field(db_path, method): + app = Datasette( + [db_path], + metadata=plugin_metadata( + { + "name_field": "name", + "description_field": "status", + } + ), + ).app() + async with httpx.AsyncClient(app=app) as client: + response = await do_method( + client, + method, + "http://localhost/test/dogs/-/reconcile", + data={"queries": json.dumps({"q0": {"query": "fido"}})}, + ) + assert 200 == response.status_code + data = response.json() + assert "q0" in data.keys() + assert len(data["q0"]["result"]) == 1 + result = data["q0"]["result"][0] + assert result["id"] == "3" + assert result["name"] == "Fido" + assert result["score"] == 100 + assert result["description"] == "bad dog" + assert response.headers["Access-Control-Allow-Origin"] == "*" diff --git a/tests/test_reconcile_config.py b/tests/test_reconcile_config.py index de54cd0..f90d376 100644 --- a/tests/test_reconcile_config.py +++ b/tests/test_reconcile_config.py @@ -148,7 +148,7 @@ async def test_plugin_configuration_use_type_default(ds): @pytest.mark.asyncio -async def test_plugin_configuration_use_fts_table(ds): +async def test_plugin_configuration_use_fts_table_none(ds): config = await check_config( { "name_field": "name", @@ -166,6 +166,85 @@ async def test_plugin_configuration_use_fts_table(ds): assert config["fts_table"] is None +@pytest.mark.asyncio +async def test_plugin_configuration_use_fts_table_config(ds): + config = await check_config( + { + "name_field": "name", + "id_field": "id", + "fts_table": "dog_fts", + "type_default": [ + { + "name": "Dog", + "id": "dog", + } + ], + }, + ds.get_database("test"), + "dogs", + ) + assert config["fts_table"] == "dog_fts" + + +@pytest.mark.asyncio +async def test_plugin_configuration_ignore_fts_table(ds_fts): + config = await check_config( + { + "name_field": "name", + "id_field": "id", + "fts_table": "dog_blah_fts", + "type_default": [ + { + "name": "Dog", + "id": "dog", + } + ], + }, + ds_fts.get_database("test"), + "dogs", + ) + assert config["fts_table"] == "dog_blah_fts" + + +@pytest.mark.asyncio +async def test_plugin_configuration_use_fts_table(ds_fts): + config = await check_config( + { + "name_field": "name", + "id_field": "id", + "type_default": [ + { + "name": "Dog", + "id": "dog", + } + ], + }, + ds_fts.get_database("test"), + "dogs", + ) + assert config["fts_table"] == "dogs_fts" + + +@pytest.mark.asyncio +async def test_plugin_configuration_use_fts_table_ignore(ds_fts): + config = await check_config( + { + "name_field": "name", + "id_field": "id", + "fts_table": None, + "type_default": [ + { + "name": "Dog", + "id": "dog", + } + ], + }, + ds_fts.get_database("test"), + "dogs", + ) + assert config["fts_table"] is None + + @pytest.mark.asyncio async def test_view_url_set(ds): config = await check_config( @@ -192,3 +271,28 @@ async def test_view_url_no_id(ds): ds.get_database("test"), "dogs", ) + + +@pytest.mark.asyncio +async def test_plugin_configuration_use_description_field(ds): + config = await check_config( + { + "name_field": "name", + "description_field": "status", + }, + ds.get_database("test"), + "dogs", + ) + assert config["description_field"] == "status" + + +@pytest.mark.asyncio +async def test_plugin_configuration_no_description_field(ds): + config = await check_config( + { + "name_field": "name", + }, + ds.get_database("test"), + "dogs", + ) + assert config["description_field"] is None diff --git a/tests/test_reconcile_schema.py b/tests/test_reconcile_schema.py index c2d95aa..f18628f 100644 --- a/tests/test_reconcile_schema.py +++ b/tests/test_reconcile_schema.py @@ -6,7 +6,7 @@ import pytest from datasette.app import Datasette -from tests.conftest import get_schema, plugin_metadata, registry +from tests.conftest import do_method, get_schema, plugin_metadata, registry logger = logging.getLogger(__name__) @@ -50,10 +50,13 @@ async def test_schema_manifest_extend(schema_version, schema, db_path): @pytest.mark.asyncio @pytest.mark.parametrize("schema_version, schema", get_schema("reconciliation-result-batch.json").items()) -async def test_response_queries_schema_post(schema_version, schema, db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_schema(schema_version, schema, db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() async with httpx.AsyncClient(app=app) as client: - response = await client.post( + response = await do_method( + client, + method, "http://localhost/test/dogs/-/reconcile", data={"queries": json.dumps({"q0": {"query": "fido"}})}, ) @@ -70,11 +73,16 @@ async def test_response_queries_schema_post(schema_version, schema, db_path): @pytest.mark.asyncio @pytest.mark.parametrize("schema_version, schema", get_schema("reconciliation-result-batch.json").items()) -async def test_response_queries_schema_get(schema_version, schema, db_path): - app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_schema_get_description_field(schema_version, schema, db_path, method): + app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "description_field": "status"})).app() async with httpx.AsyncClient(app=app) as client: - queries = json.dumps({"q0": {"query": "fido"}}) - response = await client.get(f"http://localhost/test/dogs/-/reconcile?queries={queries}") + response = await do_method( + client, + method, + "http://localhost/test/dogs/-/reconcile", + data={"queries": json.dumps({"q0": {"query": "fido"}})}, + ) assert 200 == response.status_code data = response.json() logging.info(f"Schema version: {schema_version}") @@ -88,10 +96,13 @@ async def test_response_queries_schema_get(schema_version, schema, db_path): @pytest.mark.asyncio @pytest.mark.parametrize("schema_version, schema", get_schema("reconciliation-result-batch.json").items()) -async def test_response_queries_no_results_schema_post(schema_version, schema, db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_response_queries_no_results_schema(schema_version, schema, db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() async with httpx.AsyncClient(app=app) as client: - response = await client.post( + response = await do_method( + client, + method, "http://localhost/test/dogs/-/reconcile", data={"queries": json.dumps({"q0": {"query": "abcdef"}})}, ) @@ -106,31 +117,19 @@ async def test_response_queries_no_results_schema_post(schema_version, schema, d ) -@pytest.mark.asyncio -@pytest.mark.parametrize("schema_version, schema", get_schema("reconciliation-result-batch.json").items()) -async def test_response_queries_no_results_schema_get(schema_version, schema, db_path): - app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() - async with httpx.AsyncClient(app=app) as client: - queries = json.dumps({"q0": {"query": "abcdef"}}) - response = await client.get(f"http://localhost/test/dogs/-/reconcile?queries={queries}") - assert 200 == response.status_code - data = response.json() - logging.info(f"Schema version: {schema_version}") - jsonschema.validate( - instance=data, - schema=schema, - cls=jsonschema.Draft7Validator, - registry=registry, - ) - - @pytest.mark.asyncio @pytest.mark.parametrize("schema_version, schema", get_schema("data-extension-response.json").items()) -async def test_extend_schema_post(schema_version, schema, db_path): +@pytest.mark.parametrize("method", ["post", "get"]) +async def test_extend_schema(schema_version, schema, db_path, method): app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app() async with httpx.AsyncClient(app=app) as client: extend = {"extend": json.dumps({"ids": ["1", "2", "3", "4"], "properties": [{"id": "status"}, {"id": "age"}]})} - response = await client.post("http://localhost/test/dogs/-/reconcile", data=extend) + response = await do_method( + client, + method, + "http://localhost/test/dogs/-/reconcile", + data=extend, + ) assert 200 == response.status_code data = response.json() logging.info(f"Schema version: {schema_version}") diff --git a/tests/test_reconcile_utils.py b/tests/test_reconcile_utils.py index 2f699ff..79aaca4 100644 --- a/tests/test_reconcile_utils.py +++ b/tests/test_reconcile_utils.py @@ -9,3 +9,14 @@ def test_get_select_fields(): "type_default": [{"id": "default", "name": "Default"}], } assert get_select_fields(config) == ["id", "name", "type"] + + +def test_get_select_fields_description(): + config = { + "id_field": "id", + "name_field": "name", + "type_field": "type", + "description_field": "description", + "type_default": [{"id": "default", "name": "Default"}], + } + assert get_select_fields(config) == ["id", "name", "type", "description"]