Merge branch 'develop' into redisearch-app-install
diff --git a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json
index d5fb969..e6f08f7 100644
--- a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json
+++ b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json
@@ -47,7 +47,7 @@
"item_search_settings_section",
"redisearch_warning",
"search_index_fields",
- "show_categories_in_search_autocomplete",
+ "is_redisearch_enabled",
"is_redisearch_loaded",
"shop_by_category_section",
"slideshow",
@@ -293,6 +293,7 @@
"fieldname": "search_index_fields",
"fieldtype": "Small Text",
"label": "Search Index Fields",
+ "mandatory_depends_on": "is_redisearch_enabled",
"read_only_depends_on": "eval:!doc.is_redisearch_loaded"
},
{
@@ -302,13 +303,6 @@
"label": "Item Search Settings"
},
{
- "default": "1",
- "fieldname": "show_categories_in_search_autocomplete",
- "fieldtype": "Check",
- "label": "Show Categories in Search Autocomplete",
- "read_only_depends_on": "eval:!doc.is_redisearch_loaded"
- },
- {
"default": "0",
"fieldname": "is_redisearch_loaded",
"fieldtype": "Check",
@@ -365,12 +359,19 @@
"fieldname": "show_price_in_quotation",
"fieldtype": "Check",
"label": "Show Price in Quotation"
+ },
+ {
+ "default": "0",
+ "fieldname": "is_redisearch_enabled",
+ "fieldtype": "Check",
+ "label": "Enable Redisearch",
+ "read_only_depends_on": "eval:!doc.is_redisearch_loaded"
}
],
"index_web_pages_for_search": 1,
"issingle": 1,
"links": [],
- "modified": "2021-09-02 14:02:44.785824",
+ "modified": "2022-04-01 18:35:56.106756",
"modified_by": "Administrator",
"module": "E-commerce",
"name": "E Commerce Settings",
@@ -389,5 +390,6 @@
],
"sort_field": "modified",
"sort_order": "DESC",
+ "states": [],
"track_changes": 1
}
\ No newline at end of file
diff --git a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py
index 881d833..f85667e 100644
--- a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py
+++ b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py
@@ -9,6 +9,7 @@
from erpnext.e_commerce.redisearch_utils import (
create_website_items_index,
+ define_autocomplete_dictionary,
get_indexable_web_fields,
is_search_module_loaded,
)
@@ -21,6 +22,8 @@
class ECommerceSettings(Document):
def onload(self):
self.get("__onload").quotation_series = frappe.get_meta("Quotation").get_options("naming_series")
+
+ # flag >> if redisearch is installed and loaded
self.is_redisearch_loaded = is_search_module_loaded()
def validate(self):
@@ -34,6 +37,20 @@
frappe.clear_document_cache("E Commerce Settings", "E Commerce Settings")
+ self.is_redisearch_enabled_pre_save = frappe.db.get_single_value(
+ "E Commerce Settings", "is_redisearch_enabled"
+ )
+
+ def after_save(self):
+ self.create_redisearch_indexes()
+
+ def create_redisearch_indexes(self):
+ # if redisearch is enabled (value changed) create indexes and dictionary
+ value_changed = self.is_redisearch_enabled != self.is_redisearch_enabled_pre_save
+ if self.is_redisearch_loaded and self.is_redisearch_enabled and value_changed:
+ define_autocomplete_dictionary()
+ create_website_items_index()
+
def validate_field_filters(self):
if not (self.enable_field_filters and self.filter_fields):
return
diff --git a/erpnext/e_commerce/redisearch_utils.py b/erpnext/e_commerce/redisearch_utils.py
index 82829bf..f2dd796 100644
--- a/erpnext/e_commerce/redisearch_utils.py
+++ b/erpnext/e_commerce/redisearch_utils.py
@@ -1,8 +1,12 @@
-# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt
+import json
+
import frappe
+from frappe import _
from frappe.utils.redis_wrapper import RedisWrapper
+from redis import ResponseError
from redisearch import AutoCompleter, Client, IndexDefinition, Suggestion, TagField, TextField
WEBSITE_ITEM_INDEX = "website_items_index"
@@ -22,6 +26,12 @@
return [df.fieldname for df in valid_fields]
+def is_redisearch_enabled():
+ "Return True only if redisearch is loaded and enabled."
+ is_redisearch_enabled = frappe.db.get_single_value("E Commerce Settings", "is_redisearch_enabled")
+ return is_search_module_loaded() and is_redisearch_enabled
+
+
def is_search_module_loaded():
try:
cache = frappe.cache()
@@ -32,14 +42,14 @@
)
return "search" in parsed_output
except Exception:
- return False
+ return False # handling older redis versions
-def if_redisearch_loaded(function):
- "Decorator to check if Redisearch is loaded."
+def if_redisearch_enabled(function):
+ "Decorator to check if Redisearch is enabled."
def wrapper(*args, **kwargs):
- if is_search_module_loaded():
+ if is_redisearch_enabled():
func = function(*args, **kwargs)
return func
return
@@ -51,22 +61,25 @@
return "{0}|{1}".format(frappe.conf.db_name, key).encode("utf-8")
-@if_redisearch_loaded
+@if_redisearch_enabled
def create_website_items_index():
"Creates Index Definition."
# CREATE index
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
- # DROP if already exists
try:
- client.drop_index()
- except Exception:
+ client.drop_index() # drop if already exists
+ except ResponseError:
+ # will most likely raise a ResponseError if index does not exist
+ # ignore and create index
pass
+ except Exception:
+ raise_redisearch_error()
idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)])
- # Based on e-commerce settings
+ # Index fields mentioned in e-commerce settings
idx_fields = frappe.db.get_single_value("E Commerce Settings", "search_index_fields")
idx_fields = idx_fields.split(",") if idx_fields else []
@@ -91,20 +104,20 @@
return TextField(field)
-@if_redisearch_loaded
+@if_redisearch_enabled
def insert_item_to_index(website_item_doc):
# Insert item to index
key = get_cache_key(website_item_doc.name)
cache = frappe.cache()
web_item = create_web_item_map(website_item_doc)
- for k, v in web_item.items():
- super(RedisWrapper, cache).hset(make_key(key), k, v)
+ for field, value in web_item.items():
+ super(RedisWrapper, cache).hset(make_key(key), field, value)
insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
-@if_redisearch_loaded
+@if_redisearch_enabled
def insert_to_name_ac(web_name, doc_name):
ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache())
ac.add_suggestions(Suggestion(web_name, payload=doc_name))
@@ -114,20 +127,20 @@
fields_to_index = get_fields_indexed()
web_item = {}
- for f in fields_to_index:
- web_item[f] = website_item_doc.get(f) or ""
+ for field in fields_to_index:
+ web_item[field] = website_item_doc.get(field) or ""
return web_item
-@if_redisearch_loaded
+@if_redisearch_enabled
def update_index_for_item(website_item_doc):
# Reinsert to Cache
insert_item_to_index(website_item_doc)
define_autocomplete_dictionary()
-@if_redisearch_loaded
+@if_redisearch_enabled
def delete_item_from_index(website_item_doc):
cache = frappe.cache()
key = get_cache_key(website_item_doc.name)
@@ -135,13 +148,13 @@
try:
cache.delete(key)
except Exception:
- return False
+ raise_redisearch_error()
delete_from_ac_dict(website_item_doc)
return True
-@if_redisearch_loaded
+@if_redisearch_enabled
def delete_from_ac_dict(website_item_doc):
"""Removes this items's name from autocomplete dictionary"""
cache = frappe.cache()
@@ -149,40 +162,60 @@
name_ac.delete(website_item_doc.web_item_name)
-@if_redisearch_loaded
+@if_redisearch_enabled
def define_autocomplete_dictionary():
- """Creates an autocomplete search dictionary for `name`.
- Also creats autocomplete dictionary for `categories` if
- checked in E Commerce Settings"""
+ """
+ Defines/Redefines an autocomplete search dictionary for Website Item Name.
+ Also creats autocomplete dictionary for Published Item Groups.
+ """
cache = frappe.cache()
- name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache)
- cat_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache)
-
- ac_categories = frappe.db.get_single_value(
- "E Commerce Settings", "show_categories_in_search_autocomplete"
- )
+ item_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache)
+ item_group_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache)
# Delete both autocomplete dicts
try:
cache.delete(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE))
cache.delete(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE))
except Exception:
- return False
+ raise_redisearch_error()
+ create_items_autocomplete_dict(autocompleter=item_ac)
+ create_item_groups_autocomplete_dict(autocompleter=item_group_ac)
+
+
+@if_redisearch_enabled
+def create_items_autocomplete_dict(autocompleter):
+ "Add items as suggestions in Autocompleter."
items = frappe.get_all(
"Website Item", fields=["web_item_name", "item_group"], filters={"published": 1}
)
for item in items:
- name_ac.add_suggestions(Suggestion(item.web_item_name))
- if ac_categories and item.item_group:
- cat_ac.add_suggestions(Suggestion(item.item_group))
-
- return True
+ autocompleter.add_suggestions(Suggestion(item.web_item_name))
-@if_redisearch_loaded
+@if_redisearch_enabled
+def create_item_groups_autocomplete_dict(autocompleter):
+ "Add item groups with weightage as suggestions in Autocompleter."
+ published_item_groups = frappe.get_all(
+ "Item Group", fields=["name", "route", "weightage"], filters={"show_in_website": 1}
+ )
+ if not published_item_groups:
+ return
+
+ for item_group in published_item_groups:
+ payload = json.dumps({"name": item_group.name, "route": item_group.route})
+ autocompleter.add_suggestions(
+ Suggestion(
+ string=item_group.name,
+ score=frappe.utils.flt(item_group.weightage) or 1.0,
+ payload=payload, # additional info that can be retrieved later
+ )
+ )
+
+
+@if_redisearch_enabled
def reindex_all_web_items():
items = frappe.get_all("Website Item", fields=get_fields_indexed(), filters={"published": True})
@@ -191,8 +224,8 @@
web_item = create_web_item_map(item)
key = make_key(get_cache_key(item.name))
- for k, v in web_item.items():
- super(RedisWrapper, cache).hset(key, k, v)
+ for field, value in web_item.items():
+ super(RedisWrapper, cache).hset(key, field, value)
def get_cache_key(name):
@@ -210,7 +243,12 @@
return fields_to_index
-# TODO: Remove later
-# # Figure out a way to run this at startup
-define_autocomplete_dictionary()
-create_website_items_index()
+def raise_redisearch_error():
+ "Create an Error Log and raise error."
+ traceback = frappe.get_traceback()
+ log = frappe.log_error(traceback, frappe._("Redisearch Error"))
+ log_link = frappe.utils.get_link_to_form("Error Log", log.name)
+
+ frappe.throw(
+ msg=_("Something went wrong. Check {0}").format(log_link), title=_("Redisearch Error")
+ )
diff --git a/erpnext/templates/pages/product_search.py b/erpnext/templates/pages/product_search.py
index 77a749e..3ed056f 100644
--- a/erpnext/templates/pages/product_search.py
+++ b/erpnext/templates/pages/product_search.py
@@ -1,6 +1,8 @@
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt
+import json
+
import frappe
from frappe.utils import cint, cstr
from redisearch import AutoCompleter, Client, Query
@@ -9,7 +11,7 @@
WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE,
WEBSITE_ITEM_INDEX,
WEBSITE_ITEM_NAME_AUTOCOMPLETE,
- is_search_module_loaded,
+ is_redisearch_enabled,
make_key,
)
from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website
@@ -74,8 +76,8 @@
def product_search(query, limit=10, fuzzy_search=True):
search_results = {"from_redisearch": True, "results": []}
- if not is_search_module_loaded():
- # Redisearch module not loaded
+ if not is_redisearch_enabled():
+ # Redisearch module not enabled
search_results["from_redisearch"] = False
search_results["results"] = get_product_data(query, 0, limit)
return search_results
@@ -86,6 +88,8 @@
red = frappe.cache()
query = clean_up_query(query)
+ # TODO: Check perf/correctness with Suggestions & Query vs only Query
+ # TODO: Use Levenshtein Distance in Query (max=3)
ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red)
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red)
suggestions = ac.get_suggestions(
@@ -121,8 +125,8 @@
def get_category_suggestions(query):
search_results = {"results": []}
- if not is_search_module_loaded():
- # Redisearch module not loaded, query db
+ if not is_redisearch_enabled():
+ # Redisearch module not enabled, query db
categories = frappe.db.get_all(
"Item Group",
filters={"name": ["like", "%{0}%".format(query)], "show_in_website": 1},
@@ -135,8 +139,10 @@
return search_results
ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache())
- suggestions = ac.get_suggestions(query, num=10)
+ suggestions = ac.get_suggestions(query, num=10, with_payloads=True)
- search_results["results"] = [s.string for s in suggestions]
+ results = [json.loads(s.payload) for s in suggestions]
+
+ search_results["results"] = results
return search_results