feat(pos): ability to retry on pos closing failure (#25604)

* feat(pos): ability to retry on pos closing failure

* fix: sider issues

* fix: sider issues

* fix: mark all queued closing entry as failed

* feat: add headline message
diff --git a/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.js b/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.js
index 9ea616f..aa0c53e 100644
--- a/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.js
+++ b/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.js
@@ -22,7 +22,43 @@
 		});
 		
 		if (frm.doc.docstatus === 0 && !frm.doc.amended_from) frm.set_value("period_end_date", frappe.datetime.now_datetime());
-		if (frm.doc.docstatus === 1) set_html_data(frm);
+		
+		frappe.realtime.on('closing_process_complete', async function(data) {
+			await frm.reload_doc();
+			if (frm.doc.status == 'Failed' && frm.doc.error_message && data.user == frappe.session.user) {
+				frappe.msgprint({
+					title: __('POS Closing Failed'),
+					message: frm.doc.error_message,
+					indicator: 'orange',
+					clear: true
+				});
+			}
+		});
+
+		set_html_data(frm);
+	},
+
+	refresh: function(frm) {
+		if (frm.doc.docstatus == 1 && frm.doc.status == 'Failed') {
+			const issue = '<a id="jump_to_error" style="text-decoration: underline;">issue</a>';
+			frm.dashboard.set_headline(
+				__('POS Closing failed while running in a background process. You can resolve the {0} and retry the process again.', [issue]));
+			
+			$('#jump_to_error').on('click', (e) => {
+				e.preventDefault();
+				frappe.utils.scroll_to(
+					cur_frm.get_field("error_message").$wrapper,
+					true,
+					30
+				);
+			});
+
+			frm.add_custom_button(__('Retry'), function () {
+				frm.call('retry', {}, () => {
+					frm.reload_doc();
+				});
+			});
+		}
 	},
 
 	pos_opening_entry(frm) {
@@ -61,44 +97,24 @@
 				refresh_fields(frm);
 				set_html_data(frm);
 			}
-		})
+		});
+	},
+
+	before_save: function(frm) {
+		for (let row of frm.doc.pos_transactions) {
+			frappe.db.get_doc("POS Invoice", row.pos_invoice).then(doc => {
+				cur_frm.doc.grand_total -= flt(doc.grand_total);
+				cur_frm.doc.net_total -= flt(doc.net_total);
+				cur_frm.doc.total_quantity -= flt(doc.total_qty);
+				refresh_payments(doc, cur_frm, 1);
+				refresh_taxes(doc, cur_frm, 1);
+				refresh_fields(cur_frm);
+				set_html_data(cur_frm);
+			});
+		}
 	}
 });
 
-cur_frm.cscript.before_pos_transactions_remove = function(doc, cdt, cdn) {
-	const removed_row = locals[cdt][cdn];
-
-	if (!removed_row.pos_invoice) return;
-
-	frappe.db.get_doc("POS Invoice", removed_row.pos_invoice).then(doc => {
-		cur_frm.doc.grand_total -= flt(doc.grand_total);
-		cur_frm.doc.net_total -= flt(doc.net_total);
-		cur_frm.doc.total_quantity -= flt(doc.total_qty);
-		refresh_payments(doc, cur_frm, 1);
-		refresh_taxes(doc, cur_frm, 1);
-		refresh_fields(cur_frm);
-		set_html_data(cur_frm);
-	});
-}
-
-frappe.ui.form.on('POS Invoice Reference', {
-	pos_invoice(frm, cdt, cdn) {
-		const added_row = locals[cdt][cdn];
-
-		if (!added_row.pos_invoice) return;
-
-		frappe.db.get_doc("POS Invoice", added_row.pos_invoice).then(doc => {
-			frm.doc.grand_total += flt(doc.grand_total);
-			frm.doc.net_total += flt(doc.net_total);
-			frm.doc.total_quantity += flt(doc.total_qty);
-			refresh_payments(doc, frm);
-			refresh_taxes(doc, frm);
-			refresh_fields(frm);
-			set_html_data(frm);
-		});
-	}
-})
-
 frappe.ui.form.on('POS Closing Entry Detail', {
 	closing_amount: (frm, cdt, cdn) => {
 		const row = locals[cdt][cdn];
@@ -177,11 +193,13 @@
 }
 
 function set_html_data(frm) {
-	frappe.call({
-		method: "get_payment_reconciliation_details",
-		doc: frm.doc,
-		callback: (r) => {
-			frm.get_field("payment_reconciliation_details").$wrapper.html(r.message);
-		}
-	})
+	if (frm.doc.docstatus === 1 && frm.doc.status == 'Submitted') {
+		frappe.call({
+			method: "get_payment_reconciliation_details",
+			doc: frm.doc,
+			callback: (r) => {
+				frm.get_field("payment_reconciliation_details").$wrapper.html(r.message);
+			}
+		});
+	}
 }
diff --git a/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.json b/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.json
index a9b91e0..4d6e4a2 100644
--- a/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.json
+++ b/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.json
@@ -30,6 +30,8 @@
   "total_quantity",
   "column_break_16",
   "taxes",
+  "failure_description_section",
+  "error_message",
   "section_break_14",
   "amended_from"
  ],
@@ -195,7 +197,7 @@
    "fieldtype": "Select",
    "hidden": 1,
    "label": "Status",
-   "options": "Draft\nSubmitted\nQueued\nCancelled",
+   "options": "Draft\nSubmitted\nQueued\nFailed\nCancelled",
    "print_hide": 1,
    "read_only": 1
   },
@@ -203,6 +205,21 @@
    "fieldname": "period_details_section",
    "fieldtype": "Section Break",
    "label": "Period Details"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "error_message",
+   "depends_on": "error_message",
+   "fieldname": "failure_description_section",
+   "fieldtype": "Section Break",
+   "label": "Failure Description"
+  },
+  {
+   "depends_on": "error_message",
+   "fieldname": "error_message",
+   "fieldtype": "Small Text",
+   "label": "Error",
+   "read_only": 1
   }
  ],
  "is_submittable": 1,
@@ -212,7 +229,7 @@
    "link_fieldname": "pos_closing_entry"
   }
  ],
- "modified": "2021-02-01 13:47:20.722104",
+ "modified": "2021-05-05 16:59:49.723261",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "POS Closing Entry",
diff --git a/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.py b/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.py
index 1065168..8252872 100644
--- a/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.py
+++ b/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry.py
@@ -60,6 +60,10 @@
 	def on_cancel(self):
 		unconsolidate_pos_invoices(closing_entry=self)
 
+	@frappe.whitelist()
+	def retry(self):
+		consolidate_pos_invoices(closing_entry=self)
+
 	def update_opening_entry(self, for_cancel=False):
 		opening_entry = frappe.get_doc("POS Opening Entry", self.pos_opening_entry)
 		opening_entry.pos_closing_entry = self.name if not for_cancel else None
diff --git a/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry_list.js b/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry_list.js
index 20fd610..cffeb4d 100644
--- a/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry_list.js
+++ b/erpnext/accounts/doctype/pos_closing_entry/pos_closing_entry_list.js
@@ -8,6 +8,7 @@
 			"Draft": "red",
 			"Submitted": "blue",
 			"Queued": "orange",
+			"Failed": "red",
 			"Cancelled": "red"
 
 		};
diff --git a/erpnext/accounts/doctype/pos_invoice_merge_log/pos_invoice_merge_log.py b/erpnext/accounts/doctype/pos_invoice_merge_log/pos_invoice_merge_log.py
index 4d5472d..bc78743 100644
--- a/erpnext/accounts/doctype/pos_invoice_merge_log/pos_invoice_merge_log.py
+++ b/erpnext/accounts/doctype/pos_invoice_merge_log/pos_invoice_merge_log.py
@@ -13,8 +13,7 @@
 from frappe.utils.scheduler import is_scheduler_inactive
 from frappe.core.page.background_jobs.background_jobs import get_info
 import json
-
-from six import iteritems
+import six
 
 class POSInvoiceMergeLog(Document):
 	def validate(self):
@@ -239,7 +238,7 @@
 	invoices = pos_invoices or (closing_entry and closing_entry.get('pos_transactions')) or get_all_unconsolidated_invoices()
 	invoice_by_customer = get_invoice_customer_map(invoices)
 
-	if len(invoices) >= 1 and closing_entry:
+	if len(invoices) >= 10 and closing_entry:
 		closing_entry.set_status(update=True, status='Queued')
 		enqueue_job(create_merge_logs, invoice_by_customer=invoice_by_customer, closing_entry=closing_entry)
 	else:
@@ -252,36 +251,68 @@
 		pluck='name'
 	)
 
-	if len(merge_logs) >= 1:
+	if len(merge_logs) >= 10:
 		closing_entry.set_status(update=True, status='Queued')
 		enqueue_job(cancel_merge_logs, merge_logs=merge_logs, closing_entry=closing_entry)
 	else:
 		cancel_merge_logs(merge_logs, closing_entry)
 
 def create_merge_logs(invoice_by_customer, closing_entry=None):
-	for customer, invoices in iteritems(invoice_by_customer):
-		merge_log = frappe.new_doc('POS Invoice Merge Log')
-		merge_log.posting_date = getdate(closing_entry.get('posting_date')) if closing_entry else nowdate()
-		merge_log.customer = customer
-		merge_log.pos_closing_entry = closing_entry.get('name') if closing_entry else None
+	try:
+		for customer, invoices in six.iteritems(invoice_by_customer):
+			merge_log = frappe.new_doc('POS Invoice Merge Log')
+			merge_log.posting_date = getdate(closing_entry.get('posting_date')) if closing_entry else nowdate()
+			merge_log.customer = customer
+			merge_log.pos_closing_entry = closing_entry.get('name') if closing_entry else None
 
-		merge_log.set('pos_invoices', invoices)
-		merge_log.save(ignore_permissions=True)
-		merge_log.submit()
+			merge_log.set('pos_invoices', invoices)
+			merge_log.save(ignore_permissions=True)
+			merge_log.submit()
 
-	if closing_entry:
-		closing_entry.set_status(update=True, status='Submitted')
-		closing_entry.update_opening_entry()
+		if closing_entry:
+			closing_entry.set_status(update=True, status='Submitted')
+			closing_entry.db_set('error_message', '')
+			closing_entry.update_opening_entry()
+
+	except Exception:
+		frappe.db.rollback()
+		message_log = frappe.message_log.pop()
+		error_message = safe_load_json(message_log)
+
+		if closing_entry:
+			closing_entry.set_status(update=True, status='Failed')
+			closing_entry.db_set('error_message', error_message)
+		raise
+
+	finally:
+		frappe.db.commit()
+		frappe.publish_realtime('closing_process_complete', {'user': frappe.session.user})
 
 def cancel_merge_logs(merge_logs, closing_entry=None):
-	for log in merge_logs:
-		merge_log = frappe.get_doc('POS Invoice Merge Log', log)
-		merge_log.flags.ignore_permissions = True
-		merge_log.cancel()
+	try:
+		for log in merge_logs:
+			merge_log = frappe.get_doc('POS Invoice Merge Log', log)
+			merge_log.flags.ignore_permissions = True
+			merge_log.cancel()
 
-	if closing_entry:
-		closing_entry.set_status(update=True, status='Cancelled')
-		closing_entry.update_opening_entry(for_cancel=True)
+		if closing_entry:
+			closing_entry.set_status(update=True, status='Cancelled')
+			closing_entry.db_set('error_message', '')
+			closing_entry.update_opening_entry(for_cancel=True)
+
+	except Exception:
+		frappe.db.rollback()
+		message_log = frappe.message_log.pop()
+		error_message = safe_load_json(message_log)
+
+		if closing_entry:
+			closing_entry.set_status(update=True, status='Submitted')
+			closing_entry.db_set('error_message', error_message)
+		raise
+
+	finally:
+		frappe.db.commit()
+		frappe.publish_realtime('closing_process_complete', {'user': frappe.session.user})
 
 def enqueue_job(job, **kwargs):
 	check_scheduler_status()
@@ -314,4 +345,14 @@
 def job_already_enqueued(job_name):
 	enqueued_jobs = [d.get("job_name") for d in get_info()]
 	if job_name in enqueued_jobs:
-		return True
\ No newline at end of file
+		return True
+
+def safe_load_json(message):
+	JSONDecodeError = ValueError if six.PY2 else json.JSONDecodeError
+
+	try:
+		json_message = json.loads(message).get('message')
+	except JSONDecodeError:
+		json_message = message
+
+	return json_message
\ No newline at end of file
diff --git a/erpnext/controllers/status_updater.py b/erpnext/controllers/status_updater.py
index 5276da9..4bb6138 100644
--- a/erpnext/controllers/status_updater.py
+++ b/erpnext/controllers/status_updater.py
@@ -98,6 +98,7 @@
 		["Draft", None],
 		["Submitted", "eval:self.docstatus == 1"],
 		["Queued", "eval:self.status == 'Queued'"],
+		["Failed", "eval:self.status == 'Failed'"],
 		["Cancelled", "eval:self.docstatus == 2"],
 	]
 }
diff --git a/erpnext/patches.txt b/erpnext/patches.txt
index de9f6e3..9ef949c 100644
--- a/erpnext/patches.txt
+++ b/erpnext/patches.txt
@@ -774,3 +774,4 @@
 erpnext.patches.v13_0.make_non_standard_user_type #13-04-2021
 erpnext.patches.v13_0.update_shipment_status
 erpnext.patches.v13_0.remove_attribute_field_from_item_variant_setting
+erpnext.patches.v13_0.set_pos_closing_as_failed
diff --git a/erpnext/patches/v13_0/set_pos_closing_as_failed.py b/erpnext/patches/v13_0/set_pos_closing_as_failed.py
new file mode 100644
index 0000000..1c576db
--- /dev/null
+++ b/erpnext/patches/v13_0/set_pos_closing_as_failed.py
@@ -0,0 +1,7 @@
+from __future__ import unicode_literals
+import frappe
+
+def execute():
+    frappe.reload_doc('accounts', 'doctype', 'pos_closing_entry')
+
+    frappe.db.sql("update `tabPOS Closing Entry` set `status` = 'Failed' where `status` = 'Queued'")
\ No newline at end of file