feat: Added Scheduler Job to auto update statistics
- Added frequency in Video Settings
- Cron job to check half hourly job
- Hourly job to run job based on frequency
- Patch to set youtube id in video for ease in computing
diff --git a/erpnext/hooks.py b/erpnext/hooks.py
index 95a836f..db1fd2f 100644
--- a/erpnext/hooks.py
+++ b/erpnext/hooks.py
@@ -282,6 +282,11 @@
]
scheduler_events = {
+ "cron": {
+ "0/30 * * * *": [
+ "erpnext.utilities.doctype.video.video.update_youtube_data_half_hourly",
+ ]
+ },
"all": [
"erpnext.projects.doctype.project.project.project_status_update_reminder",
"erpnext.healthcare.doctype.patient_appointment.patient_appointment.send_appointment_reminder",
@@ -297,6 +302,7 @@
"erpnext.projects.doctype.project.project.collect_project_status",
"erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts",
"erpnext.support.doctype.issue.issue.set_service_level_agreement_variance",
+ "erpnext.utilities.doctype.video.video.update_youtube_data"
],
"daily": [
"erpnext.stock.reorder_item.reorder_item",
diff --git a/erpnext/patches.txt b/erpnext/patches.txt
index 3bd4169..2a52ff6 100644
--- a/erpnext/patches.txt
+++ b/erpnext/patches.txt
@@ -718,3 +718,4 @@
erpnext.patches.v12_0.update_item_tax_template_company
erpnext.patches.v13_0.move_branch_code_to_bank_account
erpnext.patches.v13_0.healthcare_lab_module_rename_doctypes
+erpnext.patches.v13_0.set_youtube_video_id
diff --git a/erpnext/patches/v13_0/set_youtube_video_id.py b/erpnext/patches/v13_0/set_youtube_video_id.py
new file mode 100644
index 0000000..8e5dd30
--- /dev/null
+++ b/erpnext/patches/v13_0/set_youtube_video_id.py
@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+import frappe
+from erpnext.utilities.doctype.video.video import get_id_from_url
+
+def execute():
+ for video in frappe.get_all("Video", fields=["name", "url", "youtube_video_id"]):
+ if video.url and not video.youtube_video_id:
+ frappe.db.set_value("Video", video.name, "youtube_video_id", get_id_from_url(video.url))
\ No newline at end of file
diff --git a/erpnext/utilities/doctype/video/video.json b/erpnext/utilities/doctype/video/video.json
index 11df56c..2a82db2 100644
--- a/erpnext/utilities/doctype/video/video.json
+++ b/erpnext/utilities/doctype/video/video.json
@@ -11,6 +11,7 @@
"title",
"provider",
"url",
+ "youtube_video_id",
"column_break_4",
"publish_date",
"duration",
@@ -118,11 +119,18 @@
"fieldname": "youtube_tracking_section",
"fieldtype": "Section Break",
"label": "Youtube Statistics"
+ },
+ {
+ "fieldname": "youtube_video_id",
+ "fieldtype": "Data",
+ "hidden": 1,
+ "label": "Youtube ID",
+ "read_only": 1
}
],
"image_field": "image",
"links": [],
- "modified": "2020-09-04 12:59:28.283622",
+ "modified": "2020-09-07 17:02:20.185794",
"modified_by": "Administrator",
"module": "Utilities",
"name": "Video",
diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py
index 2299f95..d8653f6 100644
--- a/erpnext/utilities/doctype/video/video.py
+++ b/erpnext/utilities/doctype/video/video.py
@@ -4,8 +4,8 @@
from __future__ import unicode_literals
import frappe
-import json
import re
+import pytz
from frappe.model.document import Document
from frappe import _
from six import string_types
@@ -13,20 +13,32 @@
class Video(Document):
def validate(self):
+ self.set_video_id()
self.set_youtube_statistics()
- def set_youtube_statistics(self):
- tracking_enabled = frappe.db.get_single_value("Video Settings", "enable_youtube_tracking")
- if self.provider == "YouTube" and not tracking_enabled:
+ def set_video_id(self):
+ if self.provider == "YouTube" and self.url and not self.get("youtube_video_id"):
+ self.youtube_video_id = get_id_from_url(self.url)
+
+ @classmethod
+ def set_youtube_statistics(self, video_ids=None, update=True):
+ if self.provider == "YouTube" and not is_tracking_enabled():
return
api_key = frappe.db.get_single_value("Video Settings", "api_key")
- youtube_id = get_id_from_url(self.url)
api = Api(api_key=api_key)
try:
- video = api.get_video_by_id(video_id=youtube_id)
- video_stats = video.items[0].to_dict().get('statistics')
+ video_id = video_ids or self.youtube_video_id
+ video = api.get_video_by_id(video_id=video_id)
+
+ if video_ids:
+ video_stats = video.items
+ else:
+ video_stats = video.items[0].to_dict().get('statistics')
+
+ if not update:
+ return video_stats
self.like_count = video_stats.get('likeCount')
self.view_count = video_stats.get('viewCount')
@@ -37,16 +49,106 @@
title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name)
frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title)
+def is_tracking_enabled():
+ return frappe.db.get_single_value("Video Settings", "enable_youtube_tracking")
+
+def get_frequency(value):
+ if not value:
+ return None
+
+ # Return frequency in hours
+ if value != "Daily":
+ return frappe.utils.cint(value[:2].strip())
+ else:
+ # 24 hours for Daily
+ return 24
+
+
+def update_youtube_data_half_hourly():
+ # Called every 30 mins via hooks
+ frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency"))
+ if not is_tracking_enabled() or not frequency:
+ return
+
+ if frequency == 30:
+ batch_update_data()
+
+
+def update_youtube_data():
+ # Called every hour via hooks
+ frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency"))
+
+ # if frequency is 30 mins dont proceed, as its handled in another method
+ if not is_tracking_enabled() or not frequency or frequency == 30:
+ return
+
+ time = datetime.now()
+ timezone = pytz.timezone(frappe.utils.get_time_zone())
+ site_time = time.astimezone(timezone)
+
+ if site_time.hour % frequency == 0:
+ batch_update_youtube_data()
+
+def get_formatted_ids(video_list):
+ # format ids to comma separated string for bulk request
+ ids = []
+ for video in video_list:
+ ids.append(video.youtube_video_id)
+
+ return ','.join(ids)
+
@frappe.whitelist()
def get_id_from_url(url):
- '''
+ """
Returns video id from url
-
:param youtube url: String URL
- '''
+ """
if not isinstance(url, string_types):
frappe.throw(_("URL can only be a string"), title=_("Invalid URL"))
pattern = re.compile(r'[a-z\:\//\.]+(youtube|youtu)\.(com|be)/(watch\?v=|embed/|.+\?v=)?([^"&?\s]{11})?')
id = pattern.match(url)
- return id.groups()[-1]
\ No newline at end of file
+ return id.groups()[-1]
+
+
+@frappe.whitelist()
+def batch_update_youtube_data():
+ def prepare_and_set_data(video_list):
+ video_ids = get_formatted_ids(video_list)
+ Video.provider = "YouTube"
+ stats = Video.set_youtube_statistics(video_ids=video_ids, update=False)
+ set_youtube_data(stats)
+
+ def set_youtube_data(entries):
+ for entry in entries:
+ video_stats = entry.to_dict().get('statistics')
+ video_id = entry.to_dict().get('id')
+ stats = {
+ 'like_count' : video_stats.get('likeCount'),
+ 'view_count' : video_stats.get('viewCount'),
+ 'dislike_count' : video_stats.get('dislikeCount'),
+ 'comment_count' : video_stats.get('commentCount')
+ }
+
+ frappe.db.sql("""
+ UPDATE `tabVideo`
+ SET
+ like_count = %(like_count)s,
+ view_count = %(view_count)s,
+ dislike_count = %(dislike_count)s,
+ comment_count = %(comment_count)s
+ WHERE youtube_video_id = '{0}'""".format(video_id), stats)
+
+ frappe.log_error("yooooooooo")
+
+ video_list = frappe.get_all("Video", fields=["youtube_video_id"])
+ if len(video_list) > 50:
+ # Update in batches of 50
+ start, end = 0, 50
+ while start < len(video_list):
+ batch = video_list[start:end]
+ prepare_and_set_data(batch)
+ start += 50
+ end += 50
+ else:
+ prepare_and_set_data(video_list)
\ No newline at end of file
diff --git a/erpnext/utilities/doctype/video_settings/video_settings.json b/erpnext/utilities/doctype/video_settings/video_settings.json
index 0a0efd9..fb3274d 100644
--- a/erpnext/utilities/doctype/video_settings/video_settings.json
+++ b/erpnext/utilities/doctype/video_settings/video_settings.json
@@ -6,7 +6,8 @@
"engine": "InnoDB",
"field_order": [
"enable_youtube_tracking",
- "api_key"
+ "api_key",
+ "frequency"
],
"fields": [
{
@@ -21,11 +22,21 @@
"fieldtype": "Data",
"label": "API Key",
"mandatory_depends_on": "eval:doc.enable_youtube_tracking"
+ },
+ {
+ "default": "1 hr",
+ "depends_on": "eval:doc.enable_youtube_tracking",
+ "fieldname": "frequency",
+ "fieldtype": "Select",
+ "label": "Frequency",
+ "mandatory_depends_on": "eval:doc.enable_youtube_tracking",
+ "options": "30 mins\n1 hr\n6 hrs\nDaily"
}
],
+ "index_web_pages_for_search": 1,
"issingle": 1,
"links": [],
- "modified": "2020-08-02 03:56:49.673870",
+ "modified": "2020-09-07 16:09:00.360668",
"modified_by": "Administrator",
"module": "Utilities",
"name": "Video Settings",