From 25f6d74a4505ff26e2f8a09cf9648f77580e5693 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Nov 2015 21:16:45 +0100 Subject: [PATCH 1/4] Use youtube-dl as a library --- steve/scrapers.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/steve/scrapers.py b/steve/scrapers.py index 1a7c455..f2e09fd 100644 --- a/steve/scrapers.py +++ b/steve/scrapers.py @@ -27,7 +27,7 @@ class YoutubeScraper(object): def transform_item(self, item): """Converts youtube-dl output to richard fields""" return { - 'title': item['fulltitle'], + 'title': item['title'], 'summary': item['description'], 'description': '', 'state': 2, # Draft @@ -41,7 +41,7 @@ def transform_item(self, item): 'whiteboard': '', 'recorded': datetime.strptime(item['upload_date'], '%Y%m%d'), 'slug': '', - 'tags': item['categories'], + 'tags': item['tags'], 'speakers': [] } @@ -50,23 +50,14 @@ def scrape(self, url): if not is_youtube(url): return - # FIXME: Sometimes youtube-dl takes a *long* time to run. This - # needs to give indication of progress. - try: - output = subprocess.check_output( - ['youtube-dl', '-j', url], - stderr=subprocess.STDOUT - ) - except subprocess.CalledProcessError as cpe: - raise ScraperError('youtube-dl said "{0}".'.format(cpe.output)) - except OSError: - raise ScraperError('youtube-dl not installed or not on PATH.') + import youtube_dl - # Each line is a single JSON object. - items = [] - for line in output.splitlines(): - items.append(json.loads(line)) + ydl = youtube_dl.YoutubeDL() - items = [self.transform_item(item) for item in items] + with ydl: + result = ydl.extract_info(url, download=False) - return items + if 'entries' in result: + return [self.transform_item(item) for item in result['entries']] + else: + return [self.transform_item(result)] From e54e761ce3a7a2b46a37f7dd82c84108a481c99e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Nov 2015 21:26:15 +0100 Subject: [PATCH 2/4] Remove unused imports --- steve/scrapers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/steve/scrapers.py b/steve/scrapers.py index f2e09fd..d46beae 100644 --- a/steve/scrapers.py +++ b/steve/scrapers.py @@ -6,8 +6,6 @@ # license. ####################################################################### -import json -import subprocess from datetime import datetime from steve.util import is_youtube From 82a5d88a17b2cd468c669e73560fd6ee763ba0f3 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Nov 2015 21:44:22 +0100 Subject: [PATCH 3/4] Move imports --- steve/scrapers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/steve/scrapers.py b/steve/scrapers.py index d46beae..d032e88 100644 --- a/steve/scrapers.py +++ b/steve/scrapers.py @@ -8,6 +8,8 @@ from datetime import datetime +import youtube_dl + from steve.util import is_youtube @@ -48,11 +50,7 @@ def scrape(self, url): if not is_youtube(url): return - import youtube_dl - - ydl = youtube_dl.YoutubeDL() - - with ydl: + with youtube_dl.YoutubeDL() as ydl: result = ydl.extract_info(url, download=False) if 'entries' in result: From 9b4edc18174cb24f8713e0e0803aa4fe574a1b78 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Nov 2015 22:04:53 +0100 Subject: [PATCH 4/4] Keep compatibility with previous field names --- steve/scrapers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/steve/scrapers.py b/steve/scrapers.py index d032e88..4784dcc 100644 --- a/steve/scrapers.py +++ b/steve/scrapers.py @@ -27,7 +27,7 @@ class YoutubeScraper(object): def transform_item(self, item): """Converts youtube-dl output to richard fields""" return { - 'title': item['title'], + 'title': item.get('fulltitle') or item['title'], 'summary': item['description'], 'description': '', 'state': 2, # Draft @@ -41,7 +41,7 @@ def transform_item(self, item): 'whiteboard': '', 'recorded': datetime.strptime(item['upload_date'], '%Y%m%d'), 'slug': '', - 'tags': item['tags'], + 'tags': item.get('categories', []) + item.get('tags', []), 'speakers': [] }