refactor!: drop redisearch
incr: replace text and tag fields
incr: use rediswrapper's make key
incr: indexDefinition from redis
incr: replace index creation
incr: replace AutoCompleter
incr: replace product search ac
incr: replace client querying
fix: broken redisearch load test
fix: pass actual query to get suggestion
diff --git a/erpnext/e_commerce/redisearch_utils.py b/erpnext/e_commerce/redisearch_utils.py
index 1f649c7..87ca9bd 100644
--- a/erpnext/e_commerce/redisearch_utils.py
+++ b/erpnext/e_commerce/redisearch_utils.py
@@ -7,7 +7,9 @@
from frappe import _
from frappe.utils.redis_wrapper import RedisWrapper
from redis import ResponseError
-from redisearch import AutoCompleter, Client, IndexDefinition, Suggestion, TagField, TextField
+from redis.commands.search.field import TagField, TextField
+from redis.commands.search.indexDefinition import IndexDefinition
+from redis.commands.search.suggestion import Suggestion
WEBSITE_ITEM_INDEX = "website_items_index"
WEBSITE_ITEM_KEY_PREFIX = "website_item:"
@@ -35,12 +37,9 @@
def is_search_module_loaded():
try:
cache = frappe.cache()
- out = cache.execute_command("MODULE LIST")
-
- parsed_output = " ".join(
- (" ".join([frappe.as_unicode(s) for s in o if not isinstance(s, int)]) for o in out)
- )
- return "search" in parsed_output
+ for module in cache.module_list():
+ if module.get(b"name") == b"search":
+ return True
except Exception:
return False # handling older redis versions
@@ -58,18 +57,18 @@
def make_key(key):
- return "{0}|{1}".format(frappe.conf.db_name, key).encode("utf-8")
+ return frappe.cache().make_key(key)
@if_redisearch_enabled
def create_website_items_index():
"Creates Index Definition."
- # CREATE index
- client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
+ redis = frappe.cache()
+ index = redis.ft(WEBSITE_ITEM_INDEX)
try:
- client.drop_index() # drop if already exists
+ index.dropindex() # drop if already exists
except ResponseError:
# will most likely raise a ResponseError if index does not exist
# ignore and create index
@@ -86,9 +85,10 @@
if "web_item_name" in idx_fields:
idx_fields.remove("web_item_name")
- idx_fields = list(map(to_search_field, idx_fields))
+ idx_fields = [to_search_field(f) for f in idx_fields]
- client.create_index(
+ # TODO: sortable?
+ index.create_index(
[TextField("web_item_name", sortable=True)] + idx_fields,
definition=idx_def,
)
@@ -119,8 +119,8 @@
@if_redisearch_enabled
def insert_to_name_ac(web_name, doc_name):
- ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache())
- ac.add_suggestions(Suggestion(web_name, payload=doc_name))
+ ac = frappe.cache().ft()
+ ac.sugadd(WEBSITE_ITEM_NAME_AUTOCOMPLETE, Suggestion(web_name, payload=doc_name))
def create_web_item_map(website_item_doc):
@@ -157,9 +157,8 @@
@if_redisearch_enabled
def delete_from_ac_dict(website_item_doc):
"""Removes this items's name from autocomplete dictionary"""
- cache = frappe.cache()
- name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache)
- name_ac.delete(website_item_doc.web_item_name)
+ ac = frappe.cache().ft()
+ ac.sugdel(website_item_doc.web_item_name)
@if_redisearch_enabled
@@ -170,8 +169,6 @@
"""
cache = frappe.cache()
- item_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache)
- item_group_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache)
# Delete both autocomplete dicts
try:
@@ -180,38 +177,43 @@
except Exception:
raise_redisearch_error()
- create_items_autocomplete_dict(autocompleter=item_ac)
- create_item_groups_autocomplete_dict(autocompleter=item_group_ac)
+ create_items_autocomplete_dict()
+ create_item_groups_autocomplete_dict()
@if_redisearch_enabled
-def create_items_autocomplete_dict(autocompleter):
+def create_items_autocomplete_dict():
"Add items as suggestions in Autocompleter."
+
+ ac = frappe.cache().ft()
items = frappe.get_all(
"Website Item", fields=["web_item_name", "item_group"], filters={"published": 1}
)
-
for item in items:
- autocompleter.add_suggestions(Suggestion(item.web_item_name))
+ ac.sugadd(WEBSITE_ITEM_NAME_AUTOCOMPLETE, Suggestion(item.web_item_name))
@if_redisearch_enabled
-def create_item_groups_autocomplete_dict(autocompleter):
+def create_item_groups_autocomplete_dict():
"Add item groups with weightage as suggestions in Autocompleter."
+
published_item_groups = frappe.get_all(
"Item Group", fields=["name", "route", "weightage"], filters={"show_in_website": 1}
)
if not published_item_groups:
return
+ ac = frappe.cache().ft()
+
for item_group in published_item_groups:
payload = json.dumps({"name": item_group.name, "route": item_group.route})
- autocompleter.add_suggestions(
+ ac.sugadd(
+ WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE,
Suggestion(
string=item_group.name,
score=frappe.utils.flt(item_group.weightage) or 1.0,
payload=payload, # additional info that can be retrieved later
- )
+ ),
)
diff --git a/erpnext/templates/pages/product_search.py b/erpnext/templates/pages/product_search.py
index 0768cc3..f40fd47 100644
--- a/erpnext/templates/pages/product_search.py
+++ b/erpnext/templates/pages/product_search.py
@@ -5,14 +5,13 @@
import frappe
from frappe.utils import cint, cstr
-from redisearch import AutoCompleter, Client, Query
+from redis.commands.search.query import Query
from erpnext.e_commerce.redisearch_utils import (
WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE,
WEBSITE_ITEM_INDEX,
WEBSITE_ITEM_NAME_AUTOCOMPLETE,
is_redisearch_enabled,
- make_key,
)
from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website
from erpnext.setup.doctype.item_group.item_group import get_item_for_list_in_html
@@ -88,15 +87,17 @@
if not query:
return search_results
- red = frappe.cache()
+ redis = frappe.cache()
query = clean_up_query(query)
# TODO: Check perf/correctness with Suggestions & Query vs only Query
# TODO: Use Levenshtein Distance in Query (max=3)
- ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red)
- client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red)
- suggestions = ac.get_suggestions(
- query, num=limit, fuzzy=fuzzy_search and len(query) > 3 # Fuzzy on length < 3 can be real slow
+ redisearch = redis.ft(WEBSITE_ITEM_INDEX)
+ suggestions = redisearch.sugget(
+ WEBSITE_ITEM_NAME_AUTOCOMPLETE,
+ query,
+ num=limit,
+ fuzzy=fuzzy_search and len(query) > 3,
)
# Build a query
@@ -106,8 +107,8 @@
query_string += f"|('{clean_up_query(s.string)}')"
q = Query(query_string)
+ results = redisearch.search(q)
- results = client.search(q)
search_results["results"] = list(map(convert_to_dict, results.docs))
search_results["results"] = sorted(
search_results["results"], key=lambda k: frappe.utils.cint(k["ranking"]), reverse=True
@@ -141,8 +142,8 @@
if not query:
return search_results
- ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache())
- suggestions = ac.get_suggestions(query, num=10, with_payloads=True)
+ ac = frappe.cache().ft()
+ suggestions = ac.sugget(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE, query, num=10, with_payloads=True)
results = [json.loads(s.payload) for s in suggestions]
diff --git a/pyproject.toml b/pyproject.toml
index 5acfd39..14684f3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,7 +12,6 @@
"pycountry~=20.7.3",
"python-stdnum~=1.16",
"Unidecode~=1.2.0",
- "redisearch~=2.1.0",
# integration dependencies
"gocardless-pro~=1.22.0",