Merge pull request #15985 from adityahase/perf-patch
perf(patch): Use INSERT with ON DUPLICATE KEY UPDATE
diff --git a/erpnext/patches/v11_0/update_total_qty_field.py b/erpnext/patches/v11_0/update_total_qty_field.py
index 5c7663d..8f08699 100644
--- a/erpnext/patches/v11_0/update_total_qty_field.py
+++ b/erpnext/patches/v11_0/update_total_qty_field.py
@@ -9,12 +9,12 @@
frappe.reload_doc('stock', 'doctype', 'purchase_receipt')
frappe.reload_doc('accounts', 'doctype', 'sales_invoice')
frappe.reload_doc('accounts', 'doctype', 'purchase_invoice')
-
+
doctypes = ["Sales Order", "Sales Invoice", "Delivery Note",\
"Purchase Order", "Purchase Invoice", "Purchase Receipt", "Quotation", "Supplier Quotation"]
for doctype in doctypes:
- total_qty = frappe.db.sql('''
+ total_qty = frappe.db.sql('''
SELECT
parent, SUM(qty) as qty
FROM
@@ -22,14 +22,25 @@
GROUP BY parent
''' % (doctype), as_dict = True)
- when_then = []
- for d in total_qty:
- when_then.append("""
- when dt.name = '{0}' then {1}
- """.format(frappe.db.escape(d.get("parent")), d.get("qty")))
+ # Query to update total_qty might become too big, Update in batches
+ # batch_size is chosen arbitrarily, Don't try too hard to reason about it
+ batch_size = 100000
+ for i in range(0, len(total_qty), batch_size):
+ batch_transactions = total_qty[i:i + batch_size]
- if when_then:
- frappe.db.sql('''
- UPDATE
- `tab%s` dt SET dt.total_qty = CASE %s ELSE 0.0 END
- ''' % (doctype, " ".join(when_then)))
\ No newline at end of file
+ # UPDATE with CASE for some reason cannot use PRIMARY INDEX,
+ # causing all rows to be examined, leading to a very slow update
+
+ # UPDATE with WHERE clause uses PRIMARY INDEX, but will lead to too many queries
+
+ # INSERT with ON DUPLICATE KEY UPDATE uses PRIMARY INDEX
+ # and can perform multiple updates per query
+ # This is probably never used anywhere else as of now, but should be
+ values = []
+ for d in batch_transactions:
+ values.append("('{}', {})".format(d.parent, d.qty))
+ conditions = ",".join(values)
+ frappe.db.sql("""
+ INSERT INTO `tab{}` (name, total_qty) VALUES {}
+ ON DUPLICATE KEY UPDATE name = VALUES(name), total_qty = VALUES(total_qty)
+ """.format(doctype, conditions))