Skip to content

Commit

Permalink
use fts_table setting if it's provided - fixes #28
Browse files Browse the repository at this point in the history
also closes Check before full text search is used #11
Allow description field #9
  • Loading branch information
drkane committed Feb 2, 2024
1 parent 9a2c690 commit 1ccd702
Show file tree
Hide file tree
Showing 8 changed files with 246 additions and 87 deletions.
2 changes: 1 addition & 1 deletion src/datasette_reconcile/__about__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2023-present David Kane <[email protected]>
#
# SPDX-License-Identifier: MIT
__version__ = "0.6.2"
__version__ = "0.6.3"
5 changes: 4 additions & 1 deletion src/datasette_reconcile/reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,13 +251,16 @@ def _get_query_result(self, row, query):
}
]

return {
result = {
"id": str(row[self.config["id_field"]]),
"name": name,
"type": type_,
"score": fuzz.ratio(name_match, query_match),
"match": name_match == query_match,
}
if self.config["description_field"]:
result["description"] = str(row[self.config["description_field"]])
return result

async def _service_manifest(self, request):
# @todo: if type_field is set then get a list of types to use in the "defaultTypes" item below.
Expand Down
7 changes: 6 additions & 1 deletion src/datasette_reconcile/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ async def check_config(config, db, table):
if "name_field" not in config:
msg = "Name field must be defined to activate reconciliation"
raise ReconcileError(msg)
if "description_field" not in config:
config["description_field"] = None
if "type_field" not in config and "type_default" not in config:
config["type_default"] = [DEFAULT_TYPE]

Expand All @@ -87,7 +89,8 @@ async def check_config(config, db, table):
msg = "View URL must contain {{id}}"
raise ReconcileError(msg)

config["fts_table"] = await db.fts_table(table)
if "fts_table" not in config:
config["fts_table"] = await db.fts_table(table)

# let's show a warning if sqlite3 version is less than 3.30.0
# full text search results will fail for < 3.30.0 if the table
Expand All @@ -111,6 +114,8 @@ def get_select_fields(config):
select_fields = [config["id_field"], config["name_field"], *config.get("additional_fields", [])]
if config.get("type_field"):
select_fields.append(config["type_field"])
if config.get("description_field"):
select_fields.append(config["description_field"])
return select_fields


Expand Down
28 changes: 25 additions & 3 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
)


def create_db(tmp_path_factory):
def create_db(tmp_path_factory, enable_fts):
db_directory = tmp_path_factory.mktemp("dbs")
db_path = db_directory / "test.db"
db = sqlite_utils.Database(db_path)
Expand All @@ -29,6 +29,10 @@ def create_db(tmp_path_factory):
],
pk="id",
)

if enable_fts:
db["dogs"].enable_fts(["name"])

return db_path


Expand All @@ -55,13 +59,24 @@ def get_schema(filename):

@pytest.fixture(scope="session")
def ds(tmp_path_factory):
ds = Datasette([create_db(tmp_path_factory)], metadata=plugin_metadata())
ds = Datasette([create_db(tmp_path_factory, False)], metadata=plugin_metadata())
return ds


@pytest.fixture(scope="session")
def db_path(tmp_path_factory):
return create_db(tmp_path_factory)
return create_db(tmp_path_factory, False)


@pytest.fixture(scope="session")
def ds_fts(tmp_path_factory):
ds = Datasette([create_db(tmp_path_factory, True)], metadata=plugin_metadata())
return ds


@pytest.fixture(scope="session")
def db_path_fts(tmp_path_factory):
return create_db(tmp_path_factory, True)


def retrieve_schema_from_filesystem(uri: str):
Expand All @@ -79,3 +94,10 @@ def retrieve_schema_from_filesystem(uri: str):


registry = Registry(retrieve=retrieve_schema_from_filesystem)


def do_method(client, method, *args, **kwargs):
if method == "post":
return client.post(*args, **kwargs)
kwargs["params"] = kwargs.pop("data")
return client.get(*args, **kwargs)
117 changes: 66 additions & 51 deletions tests/test_reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import pytest
from datasette.app import Datasette

from tests.conftest import plugin_metadata
from tests.conftest import do_method, plugin_metadata


@pytest.mark.asyncio
Expand Down Expand Up @@ -123,10 +123,13 @@ async def test_servce_manifest_view_suggest(db_path, suggest_type):


@pytest.mark.asyncio
async def test_response_queries_post(db_path):
@pytest.mark.parametrize("method", ["post", "get"])
async def test_response_queries(db_path, method):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app()
async with httpx.AsyncClient(app=app) as client:
response = await client.post(
response = await do_method(
client,
method,
"http://localhost/test/dogs/-/reconcile",
data={"queries": json.dumps({"q0": {"query": "fido"}})},
)
Expand All @@ -144,37 +147,18 @@ async def test_response_queries_post(db_path):
"id": "object",
}
]
assert "description" not in result
assert response.headers["Access-Control-Allow-Origin"] == "*"


@pytest.mark.asyncio
async def test_response_queries_get(db_path):
@pytest.mark.parametrize("method", ["post", "get"])
async def test_response_queries_no_results(db_path, method):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app()
async with httpx.AsyncClient(app=app) as client:
queries = json.dumps({"q0": {"query": "fido"}})
response = await client.get(f"http://localhost/test/dogs/-/reconcile?queries={queries}")
assert 200 == response.status_code
data = response.json()
assert "q0" in data.keys()
assert len(data["q0"]["result"]) == 1
result = data["q0"]["result"][0]
assert result["id"] == "3"
assert result["name"] == "Fido"
assert result["score"] == 100
assert result["type"] == [
{
"name": "Object",
"id": "object",
}
]
assert response.headers["Access-Control-Allow-Origin"] == "*"


@pytest.mark.asyncio
async def test_response_queries_no_results_post(db_path):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app()
async with httpx.AsyncClient(app=app) as client:
response = await client.post(
response = await do_method(
client,
method,
"http://localhost/test/dogs/-/reconcile",
data={"queries": json.dumps({"q0": {"query": "abcdef"}})},
)
Expand All @@ -185,19 +169,6 @@ async def test_response_queries_no_results_post(db_path):
assert response.headers["Access-Control-Allow-Origin"] == "*"


@pytest.mark.asyncio
async def test_response_queries_no_results_get(db_path):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app()
async with httpx.AsyncClient(app=app) as client:
queries = json.dumps({"q0": {"query": "abcdef"}})
response = await client.get(f"http://localhost/test/dogs/-/reconcile?queries={queries}")
assert 200 == response.status_code
data = response.json()
assert "q0" in data.keys()
assert len(data["q0"]["result"]) == 0
assert response.headers["Access-Control-Allow-Origin"] == "*"


@pytest.mark.asyncio
async def test_response_propose_properties(db_path):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app()
Expand All @@ -213,11 +184,12 @@ async def test_response_propose_properties(db_path):


@pytest.mark.asyncio
async def test_response_extend(db_path):
@pytest.mark.parametrize("method", ["post", "get"])
async def test_response_extend(db_path, method):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name"})).app()
async with httpx.AsyncClient(app=app) as client:
extend = {"extend": json.dumps({"ids": ["1", "2", "3", "4"], "properties": [{"id": "status"}, {"id": "age"}]})}
response = await client.post("http://localhost/test/dogs/-/reconcile", data=extend)
response = await do_method(client, method, "http://localhost/test/dogs/-/reconcile", data=extend)
assert 200 == response.status_code
data = response.json()

Expand Down Expand Up @@ -343,10 +315,13 @@ async def test_response_suggest_type_1(db_path):


@pytest.mark.asyncio
async def test_response_queries_post_type(db_path):
@pytest.mark.parametrize("method", ["post", "get"])
async def test_response_queries_post_type(db_path, method):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "type_field": "status"})).app()
async with httpx.AsyncClient(app=app) as client:
response = await client.post(
response = await do_method(
client,
method,
"http://localhost/test/dogs/-/reconcile",
data={"queries": json.dumps({"q0": {"query": "fido", "type": "bad dog"}})},
)
Expand All @@ -368,10 +343,13 @@ async def test_response_queries_post_type(db_path):


@pytest.mark.asyncio
async def test_response_queries_post_type_list(db_path):
@pytest.mark.parametrize("method", ["post", "get"])
async def test_response_queries_post_type_list(db_path, method):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "type_field": "status"})).app()
async with httpx.AsyncClient(app=app) as client:
response = await client.post(
response = await do_method(
client,
method,
"http://localhost/test/dogs/-/reconcile",
data={"queries": json.dumps({"q0": {"query": "fido", "type": ["bad dog"]}})},
)
Expand All @@ -393,10 +371,13 @@ async def test_response_queries_post_type_list(db_path):


@pytest.mark.asyncio
async def test_response_queries_post_type_empty(db_path):
@pytest.mark.parametrize("method", ["post", "get"])
async def test_response_queries_post_type_empty(db_path, method):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "type_field": "status"})).app()
async with httpx.AsyncClient(app=app) as client:
response = await client.post(
response = await do_method(
client,
method,
"http://localhost/test/dogs/-/reconcile",
data={"queries": json.dumps({"q0": {"query": "fido", "type": ["good dog"]}})},
)
Expand All @@ -408,10 +389,13 @@ async def test_response_queries_post_type_empty(db_path):


@pytest.mark.asyncio
async def test_response_queries_post_type_not_given(db_path):
@pytest.mark.parametrize("method", ["post", "get"])
async def test_response_queries_post_type_not_given(db_path, method):
app = Datasette([db_path], metadata=plugin_metadata({"name_field": "name", "type_field": "status"})).app()
async with httpx.AsyncClient(app=app) as client:
response = await client.post(
response = await do_method(
client,
method,
"http://localhost/test/dogs/-/reconcile",
data={"queries": json.dumps({"q0": {"query": "fido"}})},
)
Expand All @@ -430,3 +414,34 @@ async def test_response_queries_post_type_not_given(db_path):
}
]
assert response.headers["Access-Control-Allow-Origin"] == "*"


@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["post", "get"])
async def test_response_queries_description_field(db_path, method):
app = Datasette(
[db_path],
metadata=plugin_metadata(
{
"name_field": "name",
"description_field": "status",
}
),
).app()
async with httpx.AsyncClient(app=app) as client:
response = await do_method(
client,
method,
"http://localhost/test/dogs/-/reconcile",
data={"queries": json.dumps({"q0": {"query": "fido"}})},
)
assert 200 == response.status_code
data = response.json()
assert "q0" in data.keys()
assert len(data["q0"]["result"]) == 1
result = data["q0"]["result"][0]
assert result["id"] == "3"
assert result["name"] == "Fido"
assert result["score"] == 100
assert result["description"] == "bad dog"
assert response.headers["Access-Control-Allow-Origin"] == "*"
Loading

0 comments on commit 1ccd702

Please sign in to comment.