From 7a91d1fc43f8873e6636c316006c908d7efe2f07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:03:12 +0600
Subject: [PATCH] [crooksandliars] Improve embed extractor and remove article
extractor
---
youtube_dl/extractor/__init__.py | 2 +-
youtube_dl/extractor/crooksandliars.py | 87 +++++++++++---------------
2 files changed, 39 insertions(+), 50 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 894aa5b43..bbf3be41d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,7 +90,7 @@ from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
from .condenast import CondeNastIE
from .cracked import CrackedIE
from .criterion import CriterionIE
-from .crooksandliars import CrooksAndLiarsIE, CrooksAndLiarsArticleIE
+from .crooksandliars import CrooksAndLiarsIE
from .crunchyroll import (
CrunchyrollIE,
CrunchyrollShowPlaylistIE
diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index cee0603f4..143509004 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -1,71 +1,60 @@
from __future__ import unicode_literals
-import json
-
from .common import InfoExtractor
from ..utils import (
- mimetype2ext,
+ int_or_none,
+ qualities,
)
class CrooksAndLiarsIE(InfoExtractor):
- _VALID_URL = r'(?:https?:)?//embed.crooksandliars.com/embed/(?P<id>[A-Za-z0-9]+)(?:$|[?#])'
-
+ _VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)'
_TESTS = [{
'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
'info_dict': {
- 'id': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+ 'id': '8RUoRhRi',
+ 'ext': 'mp4',
'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
- 'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
+ 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
+ 'thumbnail': 're:^https?://.*\.jpg',
'timestamp': 1428207000,
- 'thumbnail': 'https://crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
- 'uploader': "Heather",
+ 'upload_date': '20150405',
+ 'uploader': 'Heather',
+ 'duration': 236,
}
- }]
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
-
- manifest = json.loads(self._html_search_regex(r'var manifest = ({.*?})\n', webpage, 'manifest JSON'))
-
- formats = []
- for item in manifest['flavors']:
- if not item['mime'].startswith('video/'): # XXX: or item['exclude']?
- continue
- formats.append({
- 'format_id': item['type'],
- 'ext': mimetype2ext(item['mime']),
- 'url': item['url'],
- })
-
- # XXX: manifest['url']?
- return {
- 'url': url,
- 'id': video_id,
- 'uploader': manifest['author'],
- 'title': manifest['title'],
- 'description': manifest['description'],
- 'thumbnail': self._proto_relative_url(manifest['poster']),
- 'duration': manifest['duration'],
- 'timestamp': int(manifest['created']),
- 'formats': formats,
- }
-
-class CrooksAndLiarsArticleIE(InfoExtractor):
- _VALID_URL = r'(?:https?:)?//crooksandliars.com/\d+/\d+/(?P<id>[a-z\-]+)(?:/|$)'
-
- _TESTS = [{
- 'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+ }, {
+ 'url': 'http://embed.crooksandliars.com/v/MTE3MjUtMzQ2MzA',
'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
- player_url = self._proto_relative_url(self._html_search_regex(r'<iframe src="(//embed.crooksandliars.com/.*)"', webpage, 'embedded player'))
+
+ webpage = self._download_webpage(
+ 'http://embed.crooksandliars.com/embed/%s' % video_id, video_id)
+
+ manifest = self._parse_json(
+ self._search_regex(
+ r'var\s+manifest\s*=\s*({.+?})\n', webpage, 'manifest JSON'),
+ video_id)
+
+ quality = qualities(('webm_low', 'mp4_low', 'webm_high', 'mp4_high'))
+
+ formats = [{
+ 'url': item['url'],
+ 'format_id': item['type'],
+ 'quality': quality(item['type']),
+ } for item in manifest['flavors'] if item['mime'].startswith('video/')]
+ self._sort_formats(formats)
return {
- '_type': 'url',
- 'url': player_url
+ 'url': url,
+ 'id': video_id,
+ 'title': manifest['title'],
+ 'description': manifest.get('description'),
+ 'thumbnail': self._proto_relative_url(manifest.get('poster')),
+ 'timestamp': int_or_none(manifest.get('created')),
+ 'uploader': manifest.get('author'),
+ 'duration': int_or_none(manifest.get('duration')),
+ 'formats': formats,
}