feat: Make search index fields configurable
- Move indexing logic to separate file
- Add more validation logic for 'search index fields' field
diff --git a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py
index 1fd3bfa..a028a5f 100644
--- a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py
+++ b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py
@@ -6,8 +6,9 @@
from frappe.utils import cint, comma_and
from frappe import _, msgprint
from frappe.model.document import Document
-from frappe.utils import cint
-from frappe.utils import get_datetime, get_datetime_str, now_datetime
+from frappe.utils import get_datetime, get_datetime_str, now_datetime, unique
+
+from erpnext.e_commerce.website_item_indexing import create_website_items_index, ALLOWED_INDEXABLE_FIELDS_SET
class ShoppingCartSetupError(frappe.ValidationError): pass
@@ -54,13 +55,26 @@
def validate_search_index_fields(self):
if not self.search_index_fields:
- return
-
- # Clean up
- fields = self.search_index_fields.replace(' ', '')
- fields = fields.strip(',')
+ return
- self.search_index_fields = fields
+ # Clean up
+ # Remove whitespaces
+ fields = self.search_index_fields.replace(' ', '')
+ # Remove extra ',' and remove duplicates
+ fields = unique(fields.strip(',').split(','))
+
+ # All fields should be indexable
+ if not (set(fields).issubset(ALLOWED_INDEXABLE_FIELDS_SET)):
+ invalid_fields = list(set(fields).difference(ALLOWED_INDEXABLE_FIELDS_SET))
+ num_invalid_fields = len(invalid_fields)
+ invalid_fields = comma_and(invalid_fields)
+
+ if num_invalid_fields > 1:
+ frappe.throw(_("{0} are not valid options for Search Index Field.").format(frappe.bold(invalid_fields)))
+ else:
+ frappe.throw(_("{0} is not a valid option for Search Index Field.").format(frappe.bold(invalid_fields)))
+
+ self.search_index_fields = ','.join(fields)
def validate_exchange_rates_exist(self):
"""check if exchange rates exist for all Price List currencies (to company's currency)"""
@@ -113,6 +127,15 @@
def get_shipping_rules(self, shipping_territory):
return self.get_name_from_territory(shipping_territory, "shipping_rules", "shipping_rule")
+ def on_change(self):
+ old_doc = self.get_doc_before_save()
+ old_fields = old_doc.search_index_fields
+ new_fields = self.search_index_fields
+
+ # if search index fields get changed
+ if not (new_fields == old_fields):
+ create_website_items_index()
+
def validate_cart_settings(doc, method):
frappe.get_doc("E Commerce Settings", "E Commerce Settings").run_method("validate")
diff --git a/erpnext/e_commerce/doctype/website_item/website_item.py b/erpnext/e_commerce/doctype/website_item/website_item.py
index bb0af52..58d4f24 100644
--- a/erpnext/e_commerce/doctype/website_item/website_item.py
+++ b/erpnext/e_commerce/doctype/website_item/website_item.py
@@ -17,7 +17,7 @@
from erpnext.e_commerce.doctype.item_review.item_review import get_item_reviews
# SEARCH
-from erpnext.templates.pages.product_search import (
+from erpnext.e_commerce.website_item_indexing import (
insert_item_to_index,
update_index_for_item,
delete_item_from_index
diff --git a/erpnext/e_commerce/website_item_indexing.py b/erpnext/e_commerce/website_item_indexing.py
new file mode 100644
index 0000000..0e48a2d
--- /dev/null
+++ b/erpnext/e_commerce/website_item_indexing.py
@@ -0,0 +1,167 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# License: GNU General Public License v3. See license.txt
+
+from __future__ import unicode_literals
+
+import frappe
+import redis
+
+from redisearch import (
+ Client, AutoCompleter, Query,
+ Suggestion, IndexDefinition,
+ TextField, TagField,
+ Document
+ )
+
+# GLOBAL CONSTANTS
+WEBSITE_ITEM_INDEX = 'website_items_index'
+WEBSITE_ITEM_KEY_PREFIX = 'website_item:'
+WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict'
+
+ALLOWED_INDEXABLE_FIELDS_SET = {
+ 'item_code',
+ 'item_name',
+ 'item_group',
+ 'brand',
+ 'description',
+ 'web_long_description'
+}
+
+def create_website_items_index():
+ '''Creates Index Definition'''
+ # CREATE index
+ client = Client(WEBSITE_ITEM_INDEX, port=13000)
+
+ # DROP if already exists
+ try:
+ client.drop_index()
+ except:
+ pass
+
+
+ idx_def = IndexDefinition([WEBSITE_ITEM_KEY_PREFIX])
+
+ # Based on e-commerce settings
+ idx_fields = frappe.db.get_single_value(
+ 'E Commerce Settings',
+ 'search_index_fields'
+ ).split(',')
+
+ if 'web_item_name' in idx_fields:
+ idx_fields.remove('web_item_name')
+
+ idx_fields = list(map(to_search_field, idx_fields))
+
+ client.create_index(
+ [TextField("web_item_name", sortable=True)] + idx_fields,
+ definition=idx_def
+ )
+
+ reindex_all_web_items()
+ define_autocomplete_dictionary()
+
+def to_search_field(field):
+ if field == "tags":
+ return TagField("tags", separator=",")
+
+ return TextField(field)
+
+def insert_item_to_index(website_item_doc):
+ # Insert item to index
+ key = get_cache_key(website_item_doc.name)
+ r = redis.Redis("localhost", 13000)
+ web_item = create_web_item_map(website_item_doc)
+ r.hset(key, mapping=web_item)
+ insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
+
+def insert_to_name_ac(web_name, doc_name):
+ ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
+ ac.add_suggestions(Suggestion(web_name, payload=doc_name))
+
+def create_web_item_map(website_item_doc):
+ fields_to_index = get_fields_indexed()
+
+ web_item = {}
+
+ for f in fields_to_index:
+ web_item[f] = website_item_doc.get(f) or ''
+
+ return web_item
+
+def update_index_for_item(website_item_doc):
+ # Reinsert to Cache
+ insert_item_to_index(website_item_doc)
+ define_autocomplete_dictionary()
+ # TODO: Only reindex updated items
+ create_website_items_index()
+
+def delete_item_from_index(website_item_doc):
+ r = redis.Redis("localhost", 13000)
+ key = get_cache_key(website_item_doc.name)
+
+ try:
+ r.delete(key)
+ except:
+ return False
+
+ # TODO: Also delete autocomplete suggestion
+ return True
+
+def define_autocomplete_dictionary():
+ print("Defining ac dict...")
+ # AC for name
+ # TODO: AC for category
+
+ r = redis.Redis("localhost", 13000)
+ ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
+
+ try:
+ r.delete(WEBSITE_ITEM_NAME_AUTOCOMPLETE)
+ except:
+ return False
+
+ items = frappe.get_all(
+ 'Website Item',
+ fields=['web_item_name'],
+ filters={"published": True}
+ )
+
+ for item in items:
+ print("adding suggestion: " + item.web_item_name)
+ ac.add_suggestions(Suggestion(item.web_item_name))
+
+ return True
+
+def reindex_all_web_items():
+ items = frappe.get_all(
+ 'Website Item',
+ fields=get_fields_indexed(),
+ filters={"published": True}
+ )
+
+ r = redis.Redis("localhost", 13000)
+ for item in items:
+ web_item = create_web_item_map(item)
+ key = get_cache_key(item.name)
+ print(key, web_item)
+ r.hset(key, mapping=web_item)
+
+def get_cache_key(name):
+ name = frappe.scrub(name)
+ return f"{WEBSITE_ITEM_KEY_PREFIX}{name}"
+
+def get_fields_indexed():
+ fields_to_index = frappe.db.get_single_value(
+ 'E Commerce Settings',
+ 'search_index_fields'
+ ).split(',')
+
+ mandatory_fields = ['name', 'web_item_name', 'route', 'thumbnail']
+ fields_to_index = fields_to_index + mandatory_fields
+
+ return fields_to_index
+
+# TODO: Remove later
+# # Figure out a way to run this at startup
+define_autocomplete_dictionary()
+create_website_items_index()
diff --git a/erpnext/templates/pages/product_search.py b/erpnext/templates/pages/product_search.py
index 010e7a6..d935a7a 100644
--- a/erpnext/templates/pages/product_search.py
+++ b/erpnext/templates/pages/product_search.py
@@ -1,8 +1,6 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt
-from __future__ import unicode_literals
-
import frappe
from frappe.utils import cint, cstr, nowdate
@@ -10,17 +8,8 @@
from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website
# For SEARCH -------
-import redis
-from redisearch import (
- Client, AutoCompleter, Query,
- Suggestion, IndexDefinition,
- TextField, TagField,
- Document
- )
-
-WEBSITE_ITEM_INDEX = 'website_items_index'
-WEBSITE_ITEM_KEY_PREFIX = 'website_item:'
-WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict'
+from redisearch import AutoCompleter, Client, Query
+from erpnext.e_commerce.website_item_indexing import WEBSITE_ITEM_INDEX, WEBSITE_ITEM_NAME_AUTOCOMPLETE
# -----------------
no_cache = 1
@@ -94,111 +83,3 @@
def convert_to_dict(redis_search_doc):
return redis_search_doc.__dict__
-
-def create_website_items_index():
- '''Creates Index Definition'''
- # CREATE index
- client = Client(WEBSITE_ITEM_INDEX, port=13000)
-
- # DROP if already exists
- try:
- client.drop_index()
- except:
- pass
-
-
- idx_def = IndexDefinition([WEBSITE_ITEM_KEY_PREFIX])
-
- client.create_index(
- [TextField("web_item_name", sortable=True), TagField("tags")],
- definition=idx_def
- )
-
- reindex_all_web_items()
-
-def insert_item_to_index(website_item_doc):
- # Insert item to index
- key = get_cache_key(website_item_doc.name)
- r = redis.Redis("localhost", 13000)
- web_item = create_web_item_map(website_item_doc)
- r.hset(key, mapping=web_item)
- insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
-
-def insert_to_name_ac(web_name, doc_name):
- ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
- ac.add_suggestions(Suggestion(web_name, payload=doc_name))
-
-def create_web_item_map(website_item_doc):
- web_item = {}
- web_item["web_item_name"] = website_item_doc.web_item_name
- web_item["route"] = website_item_doc.route
- web_item["thumbnail"] = website_item_doc.thumbnail or ''
- web_item["description"] = website_item_doc.description or ''
-
- return web_item
-
-def update_index_for_item(website_item_doc):
- # Reinsert to Cache
- insert_item_to_index(website_item_doc)
- define_autocomplete_dictionary()
- # TODO: Only reindex updated items
- create_website_items_index()
-
-def delete_item_from_index(website_item_doc):
- r = redis.Redis("localhost", 13000)
- key = get_cache_key(website_item_doc.name)
-
- try:
- r.delete(key)
- except:
- return False
-
- # TODO: Also delete autocomplete suggestion
- return True
-
-def define_autocomplete_dictionary():
- # AC for name
- # TODO: AC for category
-
- r = redis.Redis("localhost", 13000)
- ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
-
- try:
- r.delete(WEBSITE_ITEM_NAME_AUTOCOMPLETE)
- except:
- return False
-
- items = frappe.get_all(
- 'Website Item',
- fields=['web_item_name'],
- filters={"published": True}
- )
-
- for item in items:
- print("adding suggestion: " + item.web_item_name)
- ac.add_suggestions(Suggestion(item.web_item_name))
-
- return True
-
-def reindex_all_web_items():
- items = frappe.get_all(
- 'Website Item',
- fields=['web_item_name', 'name', 'route', 'thumbnail', 'description'],
- filters={"published": True}
- )
-
- r = redis.Redis("localhost", 13000)
- for item in items:
- web_item = create_web_item_map(item)
- key = get_cache_key(item.name)
- print(key, web_item)
- r.hset(key, mapping=web_item)
-
-def get_cache_key(name):
- name = frappe.scrub(name)
- return f"{WEBSITE_ITEM_KEY_PREFIX}{name}"
-
-# TODO: Remove later
-# Figure out a way to run this at startup
-define_autocomplete_dictionary()
-create_website_items_index()