Merge branch 'develop' into youtube-analytics
diff --git a/erpnext/hooks.py b/erpnext/hooks.py
index 463ad6c..3beae3b 100644
--- a/erpnext/hooks.py
+++ b/erpnext/hooks.py
@@ -282,6 +282,11 @@
]
scheduler_events = {
+ "cron": {
+ "0/30 * * * *": [
+ "erpnext.utilities.doctype.video.video.update_youtube_data_half_hourly",
+ ]
+ },
"all": [
"erpnext.projects.doctype.project.project.project_status_update_reminder",
"erpnext.healthcare.doctype.patient_appointment.patient_appointment.send_appointment_reminder",
@@ -297,6 +302,7 @@
"erpnext.projects.doctype.project.project.collect_project_status",
"erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts",
"erpnext.support.doctype.issue.issue.set_service_level_agreement_variance",
+ "erpnext.utilities.doctype.video.video.update_youtube_data"
],
"daily": [
"erpnext.stock.reorder_item.reorder_item",
diff --git a/erpnext/patches.txt b/erpnext/patches.txt
index aa7996e..00eb093 100644
--- a/erpnext/patches.txt
+++ b/erpnext/patches.txt
@@ -725,3 +725,4 @@
erpnext.patches.v13_0.drop_razorpay_payload_column
erpnext.patches.v13_0.update_start_end_date_for_old_shift_assignment
erpnext.patches.v13_0.setting_custom_roles_for_some_regional_reports
+erpnext.patches.v13_0.set_youtube_video_id
diff --git a/erpnext/patches/v13_0/set_youtube_video_id.py b/erpnext/patches/v13_0/set_youtube_video_id.py
new file mode 100644
index 0000000..8e5dd30
--- /dev/null
+++ b/erpnext/patches/v13_0/set_youtube_video_id.py
@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+import frappe
+from erpnext.utilities.doctype.video.video import get_id_from_url
+
+def execute():
+ for video in frappe.get_all("Video", fields=["name", "url", "youtube_video_id"]):
+ if video.url and not video.youtube_video_id:
+ frappe.db.set_value("Video", video.name, "youtube_video_id", get_id_from_url(video.url))
\ No newline at end of file
diff --git a/erpnext/utilities/doctype/video/video.js b/erpnext/utilities/doctype/video/video.js
index 056bd3c..9cb5a15 100644
--- a/erpnext/utilities/doctype/video/video.js
+++ b/erpnext/utilities/doctype/video/video.js
@@ -2,7 +2,16 @@
// For license information, please see license.txt
frappe.ui.form.on('Video', {
- // refresh: function(frm) {
+ refresh: function (frm) {
+ frm.events.toggle_youtube_statistics_section(frm);
+ frm.add_custom_button("Watch Video", () => frappe.help.show_video(frm.doc.url, frm.doc.title));
+ },
- // }
+ toggle_youtube_statistics_section: (frm) => {
+ if (frm.doc.provider === "YouTube") {
+ frappe.db.get_single_value("Video Settings", "enable_youtube_tracking").then( val => {
+ frm.toggle_display("youtube_tracking_section", val);
+ });
+ }
+ }
});
diff --git a/erpnext/utilities/doctype/video/video.json b/erpnext/utilities/doctype/video/video.json
index 5d2cc13..2a82db2 100644
--- a/erpnext/utilities/doctype/video/video.json
+++ b/erpnext/utilities/doctype/video/video.json
@@ -11,11 +11,19 @@
"title",
"provider",
"url",
+ "youtube_video_id",
"column_break_4",
"publish_date",
"duration",
+ "youtube_tracking_section",
+ "like_count",
+ "view_count",
+ "col_break",
+ "dislike_count",
+ "comment_count",
"section_break_7",
- "description"
+ "description",
+ "image"
],
"fields": [
{
@@ -37,7 +45,6 @@
{
"fieldname": "url",
"fieldtype": "Data",
- "in_list_view": 1,
"label": "URL",
"reqd": 1
},
@@ -48,11 +55,12 @@
{
"fieldname": "publish_date",
"fieldtype": "Date",
+ "in_list_view": 1,
"label": "Publish Date"
},
{
"fieldname": "duration",
- "fieldtype": "Data",
+ "fieldtype": "Duration",
"label": "Duration"
},
{
@@ -62,13 +70,67 @@
{
"fieldname": "description",
"fieldtype": "Text Editor",
- "in_list_view": 1,
"label": "Description",
"reqd": 1
+ },
+ {
+ "fieldname": "like_count",
+ "fieldtype": "Float",
+ "in_list_view": 1,
+ "label": "Likes",
+ "no_copy": 1,
+ "read_only": 1
+ },
+ {
+ "fieldname": "view_count",
+ "fieldtype": "Float",
+ "in_list_view": 1,
+ "label": "Views",
+ "no_copy": 1,
+ "read_only": 1
+ },
+ {
+ "fieldname": "col_break",
+ "fieldtype": "Column Break"
+ },
+ {
+ "fieldname": "dislike_count",
+ "fieldtype": "Float",
+ "label": "Dislikes",
+ "no_copy": 1,
+ "read_only": 1
+ },
+ {
+ "fieldname": "comment_count",
+ "fieldtype": "Float",
+ "label": "Comments",
+ "no_copy": 1,
+ "read_only": 1
+ },
+ {
+ "fieldname": "image",
+ "fieldtype": "Attach Image",
+ "hidden": 1,
+ "label": "Image",
+ "no_copy": 1
+ },
+ {
+ "depends_on": "eval:doc.provider==\"YouTube\"",
+ "fieldname": "youtube_tracking_section",
+ "fieldtype": "Section Break",
+ "label": "Youtube Statistics"
+ },
+ {
+ "fieldname": "youtube_video_id",
+ "fieldtype": "Data",
+ "hidden": 1,
+ "label": "Youtube ID",
+ "read_only": 1
}
],
+ "image_field": "image",
"links": [],
- "modified": "2020-07-21 19:29:46.603734",
+ "modified": "2020-09-07 17:02:20.185794",
"modified_by": "Administrator",
"module": "Utilities",
"name": "Video",
diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py
index 3c17b56..d8653f6 100644
--- a/erpnext/utilities/doctype/video/video.py
+++ b/erpnext/utilities/doctype/video/video.py
@@ -3,8 +3,152 @@
# For license information, please see license.txt
from __future__ import unicode_literals
-# import frappe
+import frappe
+import re
+import pytz
from frappe.model.document import Document
+from frappe import _
+from six import string_types
+from pyyoutube import Api
class Video(Document):
- pass
+ def validate(self):
+ self.set_video_id()
+ self.set_youtube_statistics()
+
+ def set_video_id(self):
+ if self.provider == "YouTube" and self.url and not self.get("youtube_video_id"):
+ self.youtube_video_id = get_id_from_url(self.url)
+
+ @classmethod
+ def set_youtube_statistics(self, video_ids=None, update=True):
+ if self.provider == "YouTube" and not is_tracking_enabled():
+ return
+
+ api_key = frappe.db.get_single_value("Video Settings", "api_key")
+ api = Api(api_key=api_key)
+
+ try:
+ video_id = video_ids or self.youtube_video_id
+ video = api.get_video_by_id(video_id=video_id)
+
+ if video_ids:
+ video_stats = video.items
+ else:
+ video_stats = video.items[0].to_dict().get('statistics')
+
+ if not update:
+ return video_stats
+
+ self.like_count = video_stats.get('likeCount')
+ self.view_count = video_stats.get('viewCount')
+ self.dislike_count = video_stats.get('dislikeCount')
+ self.comment_count = video_stats.get('commentCount')
+
+ except Exception:
+ title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name)
+ frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title)
+
+def is_tracking_enabled():
+ return frappe.db.get_single_value("Video Settings", "enable_youtube_tracking")
+
+def get_frequency(value):
+ if not value:
+ return None
+
+ # Return frequency in hours
+ if value != "Daily":
+ return frappe.utils.cint(value[:2].strip())
+ else:
+ # 24 hours for Daily
+ return 24
+
+
+def update_youtube_data_half_hourly():
+ # Called every 30 mins via hooks
+ frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency"))
+ if not is_tracking_enabled() or not frequency:
+ return
+
+ if frequency == 30:
+ batch_update_data()
+
+
+def update_youtube_data():
+ # Called every hour via hooks
+ frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency"))
+
+ # if frequency is 30 mins dont proceed, as its handled in another method
+ if not is_tracking_enabled() or not frequency or frequency == 30:
+ return
+
+ time = datetime.now()
+ timezone = pytz.timezone(frappe.utils.get_time_zone())
+ site_time = time.astimezone(timezone)
+
+ if site_time.hour % frequency == 0:
+ batch_update_youtube_data()
+
+def get_formatted_ids(video_list):
+ # format ids to comma separated string for bulk request
+ ids = []
+ for video in video_list:
+ ids.append(video.youtube_video_id)
+
+ return ','.join(ids)
+
+@frappe.whitelist()
+def get_id_from_url(url):
+ """
+ Returns video id from url
+ :param youtube url: String URL
+ """
+ if not isinstance(url, string_types):
+ frappe.throw(_("URL can only be a string"), title=_("Invalid URL"))
+
+ pattern = re.compile(r'[a-z\:\//\.]+(youtube|youtu)\.(com|be)/(watch\?v=|embed/|.+\?v=)?([^"&?\s]{11})?')
+ id = pattern.match(url)
+ return id.groups()[-1]
+
+
+@frappe.whitelist()
+def batch_update_youtube_data():
+ def prepare_and_set_data(video_list):
+ video_ids = get_formatted_ids(video_list)
+ Video.provider = "YouTube"
+ stats = Video.set_youtube_statistics(video_ids=video_ids, update=False)
+ set_youtube_data(stats)
+
+ def set_youtube_data(entries):
+ for entry in entries:
+ video_stats = entry.to_dict().get('statistics')
+ video_id = entry.to_dict().get('id')
+ stats = {
+ 'like_count' : video_stats.get('likeCount'),
+ 'view_count' : video_stats.get('viewCount'),
+ 'dislike_count' : video_stats.get('dislikeCount'),
+ 'comment_count' : video_stats.get('commentCount')
+ }
+
+ frappe.db.sql("""
+ UPDATE `tabVideo`
+ SET
+ like_count = %(like_count)s,
+ view_count = %(view_count)s,
+ dislike_count = %(dislike_count)s,
+ comment_count = %(comment_count)s
+ WHERE youtube_video_id = '{0}'""".format(video_id), stats)
+
+ frappe.log_error("yooooooooo")
+
+ video_list = frappe.get_all("Video", fields=["youtube_video_id"])
+ if len(video_list) > 50:
+ # Update in batches of 50
+ start, end = 0, 50
+ while start < len(video_list):
+ batch = video_list[start:end]
+ prepare_and_set_data(batch)
+ start += 50
+ end += 50
+ else:
+ prepare_and_set_data(video_list)
\ No newline at end of file
diff --git a/erpnext/utilities/doctype/video_settings/__init__.py b/erpnext/utilities/doctype/video_settings/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/erpnext/utilities/doctype/video_settings/__init__.py
diff --git a/erpnext/utilities/doctype/video_settings/test_video_settings.py b/erpnext/utilities/doctype/video_settings/test_video_settings.py
new file mode 100644
index 0000000..b217afe
--- /dev/null
+++ b/erpnext/utilities/doctype/video_settings/test_video_settings.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestVideoSettings(unittest.TestCase):
+ pass
diff --git a/erpnext/utilities/doctype/video_settings/video_settings.js b/erpnext/utilities/doctype/video_settings/video_settings.js
new file mode 100644
index 0000000..9ac8b9e
--- /dev/null
+++ b/erpnext/utilities/doctype/video_settings/video_settings.js
@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Video Settings', {
+ // refresh: function(frm) {
+
+ // }
+});
diff --git a/erpnext/utilities/doctype/video_settings/video_settings.json b/erpnext/utilities/doctype/video_settings/video_settings.json
new file mode 100644
index 0000000..fb3274d
--- /dev/null
+++ b/erpnext/utilities/doctype/video_settings/video_settings.json
@@ -0,0 +1,60 @@
+{
+ "actions": [],
+ "creation": "2020-08-02 03:50:21.339609",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+ "enable_youtube_tracking",
+ "api_key",
+ "frequency"
+ ],
+ "fields": [
+ {
+ "default": "0",
+ "fieldname": "enable_youtube_tracking",
+ "fieldtype": "Check",
+ "label": "Enable YouTube Tracking"
+ },
+ {
+ "depends_on": "eval:doc.enable_youtube_tracking",
+ "fieldname": "api_key",
+ "fieldtype": "Data",
+ "label": "API Key",
+ "mandatory_depends_on": "eval:doc.enable_youtube_tracking"
+ },
+ {
+ "default": "1 hr",
+ "depends_on": "eval:doc.enable_youtube_tracking",
+ "fieldname": "frequency",
+ "fieldtype": "Select",
+ "label": "Frequency",
+ "mandatory_depends_on": "eval:doc.enable_youtube_tracking",
+ "options": "30 mins\n1 hr\n6 hrs\nDaily"
+ }
+ ],
+ "index_web_pages_for_search": 1,
+ "issingle": 1,
+ "links": [],
+ "modified": "2020-09-07 16:09:00.360668",
+ "modified_by": "Administrator",
+ "module": "Utilities",
+ "name": "Video Settings",
+ "owner": "Administrator",
+ "permissions": [
+ {
+ "create": 1,
+ "delete": 1,
+ "email": 1,
+ "print": 1,
+ "read": 1,
+ "role": "System Manager",
+ "share": 1,
+ "write": 1
+ }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/erpnext/utilities/doctype/video_settings/video_settings.py b/erpnext/utilities/doctype/video_settings/video_settings.py
new file mode 100644
index 0000000..36fb54f
--- /dev/null
+++ b/erpnext/utilities/doctype/video_settings/video_settings.py
@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe import _
+from frappe.model.document import Document
+from apiclient.discovery import build
+
+class VideoSettings(Document):
+ def validate(self):
+ self.validate_youtube_api_key()
+
+ def validate_youtube_api_key(self):
+ if self.enable_youtube_tracking and self.api_key:
+ try:
+ build("youtube", "v3", developerKey=self.api_key)
+ except Exception:
+ title = _("Failed to Authenticate the API key.")
+ frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title)
+ frappe.throw(title + " Please check the error logs.", title=_("Invalid Credentials"))
\ No newline at end of file
diff --git a/erpnext/utilities/report/__init__.py b/erpnext/utilities/report/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/erpnext/utilities/report/__init__.py
diff --git a/erpnext/utilities/report/youtube_interactions/__init__.py b/erpnext/utilities/report/youtube_interactions/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/erpnext/utilities/report/youtube_interactions/__init__.py
diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.js b/erpnext/utilities/report/youtube_interactions/youtube_interactions.js
new file mode 100644
index 0000000..6e3e4e6
--- /dev/null
+++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.js
@@ -0,0 +1,20 @@
+// Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+/* eslint-disable */
+
+frappe.query_reports["YouTube Interactions"] = {
+ "filters": [
+ {
+ fieldname: "from_date",
+ label: __("From Date"),
+ fieldtype: "Date",
+ default: frappe.datetime.add_months(frappe.datetime.now_date(), -12),
+ },
+ {
+ fieldname:"to_date",
+ label: __("To Date"),
+ fieldtype: "Date",
+ default: frappe.datetime.now_date(),
+ }
+ ]
+};
diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.json b/erpnext/utilities/report/youtube_interactions/youtube_interactions.json
new file mode 100644
index 0000000..a40247b
--- /dev/null
+++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.json
@@ -0,0 +1,27 @@
+{
+ "add_total_row": 0,
+ "creation": "2020-08-02 05:05:00.457093",
+ "disable_prepared_report": 0,
+ "disabled": 0,
+ "docstatus": 0,
+ "doctype": "Report",
+ "idx": 0,
+ "is_standard": "Yes",
+ "modified": "2020-08-02 05:05:00.457093",
+ "modified_by": "Administrator",
+ "module": "Utilities",
+ "name": "YouTube Interactions",
+ "owner": "Administrator",
+ "prepared_report": 0,
+ "ref_doctype": "Video",
+ "report_name": "YouTube Interactions",
+ "report_type": "Script Report",
+ "roles": [
+ {
+ "role": "All"
+ },
+ {
+ "role": "System Manager"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.py b/erpnext/utilities/report/youtube_interactions/youtube_interactions.py
new file mode 100644
index 0000000..3516a35
--- /dev/null
+++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.py
@@ -0,0 +1,113 @@
+# Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe import _
+from frappe.utils import flt
+
+def execute(filters=None):
+ if not frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") or not filters:
+ return [], []
+
+ columns = get_columns()
+ data = get_data(filters)
+ chart_data, summary = get_chart_summary_data(data)
+ return columns, data, None, chart_data, summary
+
+def get_columns():
+ return [
+ {
+ "label": _("Published Date"),
+ "fieldname": "publish_date",
+ "fieldtype": "Date",
+ "width": 100
+ },
+ {
+ "label": _("Title"),
+ "fieldname": "title",
+ "fieldtype": "Data",
+ "width": 200
+ },
+ {
+ "label": _("Duration"),
+ "fieldname": "duration",
+ "fieldtype": "Duration",
+ "width": 100
+ },
+ {
+ "label": _("Views"),
+ "fieldname": "view_count",
+ "fieldtype": "Float",
+ "width": 200
+ },
+ {
+ "label": _("Likes"),
+ "fieldname": "like_count",
+ "fieldtype": "Float",
+ "width": 200
+ },
+ {
+ "label": _("Dislikes"),
+ "fieldname": "dislike_count",
+ "fieldtype": "Float",
+ "width": 100
+ },
+ {
+ "label": _("Comments"),
+ "fieldname": "comment_count",
+ "fieldtype": "Float",
+ "width": 100
+ }
+ ]
+
+def get_data(filters):
+ return frappe.db.sql("""
+ SELECT
+ publish_date, title, provider, duration,
+ view_count, like_count, dislike_count, comment_count
+ FROM `tabVideo`
+ WHERE view_count is not null
+ and publish_date between %(from_date)s and %(to_date)s
+ ORDER BY view_count desc""", filters, as_dict=1)
+
+def get_chart_summary_data(data):
+ labels, likes, views = [], [], []
+ total_views = 0
+
+ for row in data:
+ labels.append(row.get('title'))
+ likes.append(row.get('like_count'))
+ views.append(row.get('view_count'))
+ total_views += flt(row.get('view_count'))
+
+
+ chart_data = {
+ "data" : {
+ "labels" : labels,
+ "datasets" : [
+ {
+ "name" : "Likes",
+ "values" : likes
+ },
+ {
+ "name" : "Views",
+ "values" : views
+ }
+ ]
+ },
+ "type": "bar",
+ "barOptions": {
+ "stacked": 1
+ },
+ }
+
+ summary = [
+ {
+ "value": total_views,
+ "indicator": "Blue",
+ "label": "Total Views",
+ "datatype": "Float",
+ }
+ ]
+ return chart_data, summary
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 912d61f..872d78c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,6 +7,7 @@
pycountry==19.8.18
PyGithub==1.44.1
python-stdnum==1.12
+python-youtube==0.6.0
taxjar==1.9.0
tweepy==3.8.0
Unidecode==1.1.1