release 2014.11.02.1

[FunnyOrDie] Add support for "/articles/" URLs

README.md

@@ -381,7 +381,7 @@ Again, from then on you'll be able to update with `sudo youtube-dl -U`.
 
 YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
 
-If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to report bugs to the Ubuntu packaging guys - all they have to do is update the package to a somewhat recent version. See above for a way to update.
+If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
 ### Do I always have to pass in `--max-quality FORMAT`, or `-citw`?
 

youtube_dl/YoutubeDL.py

@@ -1211,6 +1211,8 @@ class YoutubeDL(object):
             res += 'video@'
         if fdict.get('vbr') is not None:
             res += '%4dk' % fdict['vbr']
+        if fdict.get('fps') is not None:
+            res += ', %sfps' % fdict['fps']
         if fdict.get('acodec') is not None:
             if res:
                 res += ', '

youtube_dl/extractor/__init__.py

@@ -67,7 +67,6 @@ from .crunchyroll import (
     CrunchyrollShowPlaylistIE
 )
 from .cspan import CSpanIE
-from .d8 import D8IE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,

youtube_dl/extractor/canalplus.py

@@ -7,15 +7,21 @@ from .common import InfoExtractor
 from ..utils import (
     unified_strdate,
     url_basename,
+    qualities,
 )
 
 
 class CanalplusIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.canalplus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
-    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
-    IE_NAME = 'canalplus.fr'
+    IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
+    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
+    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
+    _SITE_ID_MAP = {
+        'canalplus.fr': 'cplus',
+        'piwiplus.fr': 'teletoon',
+        'd8.tv': 'd8',
+    }
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
         'md5': '3db39fb48b9685438ecf33a1078023e4',
         'info_dict': {
@@ -25,36 +31,73 @@ class CanalplusIE(InfoExtractor):
             'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
             'upload_date': '20130826',
         },
-    }
+    }, {
+        'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
+        'info_dict': {
+            'id': '1108190',
+            'ext': 'flv',
+            'title': 'Le labyrinthe - Boing super ranger',
+            'description': 'md5:4cea7a37153be42c1ba2c1d3064376ff',
+            'upload_date': '20140724',
+        },
+        'skip': 'Only works from France',
+    }, {
+        'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
+        'info_dict': {
+            'id': '966289',
+            'ext': 'flv',
+            'title': 'Campagne intime - Documentaire exceptionnel',
+            'description': 'md5:d2643b799fb190846ae09c61e59a859f',
+            'upload_date': '20131108',
+        },
+        'skip': 'videos get deleted after a while',
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.groupdict().get('id')
 
+        site_id = self._SITE_ID_MAP[mobj.group('site') or 'canal']
+
         # Beware, some subclasses do not define an id group
         display_id = url_basename(mobj.group('path'))
 
         if video_id is None:
             webpage = self._download_webpage(url, display_id)
-            video_id = self._search_regex(r'<canal:player videoId="(\d+)"', webpage, 'video id')
+            video_id = self._search_regex(
+                r'<canal:player[^>]+?videoId="(\d+)"', webpage, 'video id')
 
-        info_url = self._VIDEO_INFO_TEMPLATE % video_id
+        info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')
 
         video_info = [video for video in doc if video.find('ID').text == video_id][0]
         media = video_info.find('MEDIA')
         infos = video_info.find('INFOS')
 
-        preferences = ['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS']
+        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS'])
 
-        formats = [
-            {
-                'url': fmt.text + '?hdcore=2.11.3' if fmt.tag == 'HDS' else fmt.text,
-                'format_id': fmt.tag,
-                'ext': 'mp4' if fmt.tag == 'HLS' else 'flv',
-                'preference': preferences.index(fmt.tag) if fmt.tag in preferences else -1,
-            } for fmt in media.find('VIDEOS') if fmt.text
-        ]
+        formats = []
+        for fmt in media.find('VIDEOS'):
+            format_url = fmt.text
+            if not format_url:
+                continue
+            format_id = fmt.tag
+            if format_id == 'HLS':
+                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'flv')
+                for fmt in hls_formats:
+                    fmt['preference'] = preference(format_id)
+                formats.extend(hls_formats)
+            elif format_id == 'HDS':
+                hds_formats = self._extract_f4m_formats(format_url + '?hdcore=2.11.3', video_id)
+                for fmt in hds_formats:
+                    fmt['preference'] = preference(format_id)
+                formats.extend(hds_formats)
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'preference': preference(format_id),
+                })
         self._sort_formats(formats)
 
         return {

youtube_dl/extractor/cinemassacre.py

@@ -59,12 +59,9 @@ class CinemassacreIE(InfoExtractor):
 
         vidurl = self._search_regex(
             r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
-        vidid = self._search_regex(
-            r'\'vidid\'\s*:\s*"([^\']+)"', playerdata, 'vidid')
-        videoserver = self._html_search_regex(
-            r"'videoserver'\s*:\s*'([^']+)'", playerdata, 'videoserver')
 
-        videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
+        videolist_url = self._search_regex(
+            r"file\s*:\s*'(http.+?/jwplayer\.smil)'", playerdata, 'jwplayer.smil')
         videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
 
         formats = []

youtube_dl/extractor/common.py

@@ -72,6 +72,7 @@ class InfoExtractor(object):
                     * acodec     Name of the audio codec in use
                     * asr        Audio sampling rate in Hertz
                     * vbr        Average video bitrate in KBit/s
+                    * fps        Frame rate
                     * vcodec     Name of the video codec in use
                     * container  Name of the container format
                     * filesize   The number of bytes, if known in advance
@@ -618,6 +619,7 @@ class InfoExtractor(object):
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
+                f.get('fps') if f.get('fps') is not None else -1,
                 f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,

youtube_dl/extractor/d8.py

@@ -1,25 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-from .canalplus import CanalplusIE
-
-
-class D8IE(CanalplusIE):
-    _VALID_URL = r'https?://www\.d8\.tv/.*?/(?P<path>.*)'
-    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/d8/%s'
-    IE_NAME = 'd8.tv'
-
-    _TEST = {
-        'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
-        'file': '966289.flv',
-        'info_dict': {
-            'title': 'Campagne intime - Documentaire exceptionnel',
-            'description': 'md5:d2643b799fb190846ae09c61e59a859f',
-            'upload_date': '20131108',
-        },
-        'params': {
-            # rtmp
-            'skip_download': True,
-        },
-        'skip': 'videos get deleted after a while',
-    }

youtube_dl/extractor/funnyordie.py

@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 
 
 class FunnyOrDieIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
+    _VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|articles|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
     _TESTS = [{
         'url': 'http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version',
         'md5': 'bcd81e0c4f26189ee09be362ad6e6ba9',
@@ -29,6 +29,9 @@ class FunnyOrDieIE(InfoExtractor):
             'description': 'Please use this to sell something.  www.jonlajoie.com',
             'thumbnail': 're:^http:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.funnyordie.com/articles/ebf5e34fc8/10-hours-of-walking-in-nyc-as-a-man',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

youtube_dl/extractor/generic.py

@@ -887,7 +887,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'SBS')
 
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://m\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')

youtube_dl/extractor/mlb.py

@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|shared/video/embed/embed\.html\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'https?://m(?:lb)?\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -72,6 +72,14 @@ class MLBIE(InfoExtractor):
             'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
             'only_matching': True,
         },
+        {
+            'url': 'http://mlb.mlb.com/shared/video/embed/embed.html?content_id=36599553',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

youtube_dl/extractor/ro220.py

@@ -1,43 +1,43 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    compat_parse_qs,
-)
+from ..utils import compat_urllib_parse_unquote
 
 
 class Ro220IE(InfoExtractor):
     IE_NAME = '220.ro'
-    _VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<video_id>[^/]+)'
+    _VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<id>[^/]+)'
     _TEST = {
-        "url": "http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/",
-        'file': 'LYV6doKo7f.mp4',
+        'url': 'http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/',
         'md5': '03af18b73a07b4088753930db7a34add',
         'info_dict': {
-            "title": "Luati-le Banii sez 4 ep 1",
-            "description": "re:^Iata-ne reveniti dupa o binemeritata vacanta\. +Va astept si pe Facebook cu pareri si comentarii.$",
+            'id': 'LYV6doKo7f',
+            'ext': 'mp4',
+            'title': 'Luati-le Banii sez 4 ep 1',
+            'description': 're:^Iata-ne reveniti dupa o binemeritata vacanta\. +Va astept si pe Facebook cu pareri si comentarii.$',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        flashVars_str = self._search_regex(
-            r'<param name="flashVars" value="([^"]+)"',
-            webpage, 'flashVars')
-        flashVars = compat_parse_qs(flashVars_str)
+        url = compat_urllib_parse_unquote(self._search_regex(
+            r'(?s)clip\s*:\s*{.*?url\s*:\s*\'([^\']+)\'', webpage, 'url'))
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        formats = [{
+            'format_id': 'sd',
+            'url': url,
+            'ext': 'mp4',
+        }]
 
         return {
-            '_type': 'video',
             'id': video_id,
-            'ext': 'mp4',
-            'url': flashVars['videoURL'][0],
-            'title': flashVars['title'][0],
-            'description': clean_html(flashVars['desc'][0]),
-            'thumbnail': flashVars['preview'][0],
+            'formats': formats,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
         }

youtube_dl/extractor/youtube.py

@@ -274,6 +274,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '138': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
@@ -297,6 +300,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.29'
+__version__ = '2014.11.02.1'