release 2013.01.17

If the url doesn’t specify the protocol we can’t directly use it to download the webpage, we would need to build a new url.
Instead, we let the generic extractor add the protocol.

README.md

@@ -58,8 +58,10 @@ which means you can modify it, redistribute it or use it however you like.
     --max-filesize SIZE        Do not download any videos larger than SIZE (e.g.
                                50k or 44.6m)
     --date DATE                download only videos uploaded in this date
-    --datebefore DATE          download only videos uploaded before this date
-    --dateafter DATE           download only videos uploaded after this date
+    --datebefore DATE          download only videos uploaded on or before this
+                               date (i.e. inclusive)
+    --dateafter DATE           download only videos uploaded on or after this
+                               date (i.e. inclusive)
     --min-views COUNT          Do not download any videos with less than COUNT
                                views
     --max-views COUNT          Do not download any videos with more than COUNT
@@ -91,13 +93,13 @@ which means you can modify it, redistribute it or use it however you like.
                                different, %(autonumber)s to get an automatically
                                incremented number, %(ext)s for the filename
                                extension, %(format)s for the format description
-                               (like "22 - 1280x720" or "HD"),%(format_id)s for
+                               (like "22 - 1280x720" or "HD"), %(format_id)s for
                                the unique id of the format (like Youtube's
-                               itags: "137"),%(upload_date)s for the upload date
-                               (YYYYMMDD), %(extractor)s for the provider
-                               (youtube, metacafe, etc), %(id)s for the video id
-                               , %(playlist)s for the playlist the video is in,
-                               %(playlist_index)s for the position in the
+                               itags: "137"), %(upload_date)s for the upload
+                               date (YYYYMMDD), %(extractor)s for the provider
+                               (youtube, metacafe, etc), %(id)s for the video
+                               id, %(playlist)s for the playlist the video is
+                               in, %(playlist_index)s for the position in the
                                playlist and %% for a literal percent. Use - to
                                output to stdout. Can also be used to download to
                                a different directory, for example with -o '/my/d
@@ -109,7 +111,7 @@ which means you can modify it, redistribute it or use it however you like.
                                avoid "&" and spaces in filenames
     -a, --batch-file FILE      file containing URLs to download ('-' for stdin)
     --load-info FILE           json file containing the video information
-                               (created with the "--write-json" option
+                               (created with the "--write-json" option)
     -w, --no-overwrites        do not overwrite files
     -c, --continue             force resume of partially downloaded files. By
                                default, youtube-dl will resume downloads if
@@ -143,7 +145,7 @@ which means you can modify it, redistribute it or use it however you like.
     --no-progress              do not print progress bar
     --console-title            display progress in console titlebar
     -v, --verbose              print various debugging information
-    --dump-intermediate-pages  print downloaded pages to debug problems(very
+    --dump-intermediate-pages  print downloaded pages to debug problems (very
                                verbose)
     --write-pages              Write downloaded intermediary pages to files in
                                the current directory to debug problems
@@ -156,8 +158,7 @@ which means you can modify it, redistribute it or use it however you like.
     --prefer-free-formats      prefer free video formats unless a specific one
                                is requested
     --max-quality FORMAT       highest quality format to download
-    -F, --list-formats         list all available formats (currently youtube
-                               only)
+    -F, --list-formats         list all available formats
 
 ## Subtitle Options:
     --write-sub                write subtitle file
@@ -175,7 +176,7 @@ which means you can modify it, redistribute it or use it however you like.
     -u, --username USERNAME    account username
     -p, --password PASSWORD    account password
     -n, --netrc                use .netrc authentication data
-    --video-password PASSWORD  video password (vimeo only)
+    --video-password PASSWORD  video password (vimeo, smotri)
 
 ## Post-processing Options:
     -x, --extract-audio        convert video files to audio-only files (requires
@@ -193,7 +194,13 @@ which means you can modify it, redistribute it or use it however you like.
                                processed files are overwritten by default
     --embed-subs               embed subtitles in the video (only for mp4
                                videos)
-    --add-metadata             add metadata to the files
+    --add-metadata             write metadata to the video file
+    --xattrs                   write metadata to the video file's xattrs (using
+                               dublin core and xdg standards)
+    --prefer-avconv            Prefer avconv over ffmpeg for running the
+                               postprocessors (default)
+    --prefer-ffmpeg            Prefer ffmpeg over avconv for running the
+                               postprocessors
 
 # CONFIGURATION
 
@@ -232,9 +239,12 @@ Videos can be filtered by their upload date using the options `--date`, `--dateb
  
 Examples:
 
-	$ youtube-dl --dateafter now-6months #will only download the videos uploaded in the last 6 months
-	$ youtube-dl --date 19700101 #will only download the videos uploaded in January 1, 1970
-	$ youtube-dl --dateafter 20000101 --datebefore 20100101 #will only download the videos uploaded between 2000 and 2010
+  $ # Download only the videos uploaded in the last 6 months
+	$ youtube-dl --dateafter now-6months
+  $ # Download only the videos uploaded on January 1, 1970
+	$ youtube-dl --date 19700101
+  $ # will only download the videos uploaded in the 200x decade
+	$ youtube-dl --dateafter 20000101 --datebefore 20091231
 
 # FAQ
 
@@ -313,7 +323,7 @@ Site support requests must contain an example URL. An example URL is a URL you m
 
 ###  Are you using the latest version?
 
-Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. Ábout 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
 
 ###  Is the issue already documented?
 

devscripts/bash-completion.in

@@ -6,7 +6,7 @@ __youtube_dl()
     prev="${COMP_WORDS[COMP_CWORD-1]}"
     opts="{{flags}}"
     keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory"
-    fileopts="-a|--batch-file|--download-archive|--cookies"
+    fileopts="-a|--batch-file|--download-archive|--cookies|--load-info"
     diropts="--cache-dir"
 
     if [[ ${prev} =~ ${fileopts} ]]; then

devscripts/check-porn.py

@@ -3,6 +3,9 @@
 """
 This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
 if we are not 'age_limit' tagging some porn site
+
+A second approach implemented relies on a list of porn domains, to activate it
+pass the list filename as the only argument
 """
 
 # Allow direct execution
@@ -11,25 +14,42 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import get_testcases
+from youtube_dl.utils import compat_urllib_parse_urlparse
 from youtube_dl.utils import compat_urllib_request
 
+if len(sys.argv) > 1:
+    METHOD = 'LIST'
+    LIST = open(sys.argv[1]).read().decode('utf8').strip()
+else:
+    METHOD = 'EURISTIC'
+
 for test in get_testcases():
-    try:
-        webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
-    except:
-        print('\nFail: {0}'.format(test['name']))
-        continue
+    if METHOD == 'EURISTIC':
+        try:
+            webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
+        except:
+            print('\nFail: {0}'.format(test['name']))
+            continue
 
-    webpage = webpage.decode('utf8', 'replace')
+        webpage = webpage.decode('utf8', 'replace')
 
-    if 'porn' in webpage.lower() and ('info_dict' not in test
-                                      or 'age_limit' not in test['info_dict']
-                                      or test['info_dict']['age_limit'] != 18):
+        RESULT = 'porn' in webpage.lower()
+
+    elif METHOD == 'LIST':
+        domain = compat_urllib_parse_urlparse(test['url']).netloc
+        if not domain:
+            print('\nFail: {0}'.format(test['name']))
+            continue
+        domain = '.'.join(domain.split('.')[-2:])
+
+        RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)
+
+    if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
+                   or test['info_dict']['age_limit'] != 18):
         print('\nPotential missing age_limit check: {0}'.format(test['name']))
 
-    elif 'porn' not in webpage.lower() and ('info_dict' in test and
-                                            'age_limit' in test['info_dict'] and
-                                            test['info_dict']['age_limit'] == 18):
+    elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
+                         and test['info_dict']['age_limit'] == 18):
         print('\nPotential false negative: {0}'.format(test['name']))
 
     else:

devscripts/release.sh

@@ -24,6 +24,8 @@ if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.0
 version="$1"
 if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
 if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+useless_files=$(find youtube_dl -type f -not -name '*.py')
+if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."

setup.py

@@ -71,7 +71,10 @@ setup(
     author_email='ytdl@yt-dl.org',
     maintainer='Philipp Hagemeister',
     maintainer_email='phihag@phihag.de',
-    packages=['youtube_dl', 'youtube_dl.extractor', 'youtube_dl.downloader'],
+    packages=[
+        'youtube_dl',
+        'youtube_dl.extractor', 'youtube_dl.downloader',
+        'youtube_dl.postprocessor'],
 
     # Provokes warning on most systems (why?!)
     # test_suite = 'nose.collector',

test/test_all_urls.py

@@ -113,6 +113,8 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_vimeo_matching(self):
         self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
         self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user'])
+        self.assertMatch('http://vimeo.com/user7108434/videos', ['vimeo:user'])
+        self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])
 
     # https://github.com/rg3/youtube-dl/issues/1930
     def test_soundcloud_not_matching_sets(self):

test/test_playlists.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # encoding: utf-8
 
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
@@ -30,6 +31,7 @@ from youtube_dl.extractor import (
     SmotriUserIE,
     IviCompilationIE,
     ImdbListIE,
+    KhanAcademyIE,
 )
 
 
@@ -43,7 +45,7 @@ class TestPlaylists(unittest.TestCase):
         ie = DailymotionPlaylistIE(dl)
         result = ie.extract('http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'SPORT')
+        self.assertEqual(result['title'], 'SPORT')
         self.assertTrue(len(result['entries']) > 20)
 
     def test_dailymotion_user(self):
@@ -51,7 +53,7 @@ class TestPlaylists(unittest.TestCase):
         ie = DailymotionUserIE(dl)
         result = ie.extract('http://www.dailymotion.com/user/generation-quoi/')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Génération Quoi')
+        self.assertEqual(result['title'], 'Génération Quoi')
         self.assertTrue(len(result['entries']) >= 26)
 
     def test_vimeo_channel(self):
@@ -59,7 +61,7 @@ class TestPlaylists(unittest.TestCase):
         ie = VimeoChannelIE(dl)
         result = ie.extract('http://vimeo.com/channels/tributes')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Vimeo Tributes')
+        self.assertEqual(result['title'], 'Vimeo Tributes')
         self.assertTrue(len(result['entries']) > 24)
 
     def test_vimeo_user(self):
@@ -67,7 +69,7 @@ class TestPlaylists(unittest.TestCase):
         ie = VimeoUserIE(dl)
         result = ie.extract('http://vimeo.com/nkistudio/videos')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Nki')
+        self.assertEqual(result['title'], 'Nki')
         self.assertTrue(len(result['entries']) > 65)
 
     def test_vimeo_album(self):
@@ -75,7 +77,7 @@ class TestPlaylists(unittest.TestCase):
         ie = VimeoAlbumIE(dl)
         result = ie.extract('http://vimeo.com/album/2632481')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Staff Favorites: November 2013')
+        self.assertEqual(result['title'], 'Staff Favorites: November 2013')
         self.assertTrue(len(result['entries']) > 12)
 
     def test_vimeo_groups(self):
@@ -83,7 +85,7 @@ class TestPlaylists(unittest.TestCase):
         ie = VimeoGroupsIE(dl)
         result = ie.extract('http://vimeo.com/groups/rolexawards')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Rolex Awards for Enterprise')
+        self.assertEqual(result['title'], 'Rolex Awards for Enterprise')
         self.assertTrue(len(result['entries']) > 72)
 
     def test_ustream_channel(self):
@@ -91,7 +93,7 @@ class TestPlaylists(unittest.TestCase):
         ie = UstreamChannelIE(dl)
         result = ie.extract('http://www.ustream.tv/channel/young-americans-for-liberty')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'5124905')
+        self.assertEqual(result['id'], '5124905')
         self.assertTrue(len(result['entries']) >= 11)
 
     def test_soundcloud_set(self):
@@ -99,7 +101,7 @@ class TestPlaylists(unittest.TestCase):
         ie = SoundcloudSetIE(dl)
         result = ie.extract('https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'The Royal Concept EP')
+        self.assertEqual(result['title'], 'The Royal Concept EP')
         self.assertTrue(len(result['entries']) >= 6)
 
     def test_soundcloud_user(self):
@@ -107,7 +109,7 @@ class TestPlaylists(unittest.TestCase):
         ie = SoundcloudUserIE(dl)
         result = ie.extract('https://soundcloud.com/the-concept-band')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'9615865')
+        self.assertEqual(result['id'], '9615865')
         self.assertTrue(len(result['entries']) >= 12)
 
     def test_livestream_event(self):
@@ -115,7 +117,7 @@ class TestPlaylists(unittest.TestCase):
         ie = LivestreamIE(dl)
         result = ie.extract('http://new.livestream.com/tedx/cityenglish')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'TEDCity2.0 (English)')
+        self.assertEqual(result['title'], 'TEDCity2.0 (English)')
         self.assertTrue(len(result['entries']) >= 4)
 
     def test_nhl_videocenter(self):
@@ -123,8 +125,8 @@ class TestPlaylists(unittest.TestCase):
         ie = NHLVideocenterIE(dl)
         result = ie.extract('http://video.canucks.nhl.com/videocenter/console?catid=999')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'999')
-        self.assertEqual(result['title'], u'Highlights')
+        self.assertEqual(result['id'], '999')
+        self.assertEqual(result['title'], 'Highlights')
         self.assertEqual(len(result['entries']), 12)
 
     def test_bambuser_channel(self):
@@ -132,7 +134,7 @@ class TestPlaylists(unittest.TestCase):
         ie = BambuserChannelIE(dl)
         result = ie.extract('http://bambuser.com/channel/pixelversity')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'pixelversity')
+        self.assertEqual(result['title'], 'pixelversity')
         self.assertTrue(len(result['entries']) >= 60)
 
     def test_bandcamp_album(self):
@@ -140,7 +142,7 @@ class TestPlaylists(unittest.TestCase):
         ie = BandcampAlbumIE(dl)
         result = ie.extract('http://mpallante.bandcamp.com/album/nightmare-night-ep')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Nightmare Night EP')
+        self.assertEqual(result['title'], 'Nightmare Night EP')
         self.assertTrue(len(result['entries']) >= 4)
         
     def test_smotri_community(self):
@@ -148,8 +150,8 @@ class TestPlaylists(unittest.TestCase):
         ie = SmotriCommunityIE(dl)
         result = ie.extract('http://smotri.com/community/video/kommuna')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'kommuna')
-        self.assertEqual(result['title'], u'КПРФ')
+        self.assertEqual(result['id'], 'kommuna')
+        self.assertEqual(result['title'], 'КПРФ')
         self.assertTrue(len(result['entries']) >= 4)
         
     def test_smotri_user(self):
@@ -157,17 +159,17 @@ class TestPlaylists(unittest.TestCase):
         ie = SmotriUserIE(dl)
         result = ie.extract('http://smotri.com/user/inspector')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'inspector')
-        self.assertEqual(result['title'], u'Inspector')
+        self.assertEqual(result['id'], 'inspector')
+        self.assertEqual(result['title'], 'Inspector')
         self.assertTrue(len(result['entries']) >= 9)
 
     def test_AcademicEarthCourse(self):
         dl = FakeYDL()
         ie = AcademicEarthCourseIE(dl)
-        result = ie.extract(u'http://academicearth.org/courses/building-dynamic-websites/')
+        result = ie.extract('http://academicearth.org/courses/building-dynamic-websites/')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'building-dynamic-websites')
-        self.assertEqual(result['title'], u'Building Dynamic Websites')
+        self.assertEqual(result['id'], 'building-dynamic-websites')
+        self.assertEqual(result['title'], 'Building Dynamic Websites')
         self.assertEqual(result['description'], u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.")
         self.assertEqual(len(result['entries']), 10)
         
@@ -176,8 +178,8 @@ class TestPlaylists(unittest.TestCase):
         ie = IviCompilationIE(dl)
         result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'dezhurnyi_angel')
-        self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012)')
+        self.assertEqual(result['id'], 'dezhurnyi_angel')
+        self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012)')
         self.assertTrue(len(result['entries']) >= 36)
         
     def test_ivi_compilation_season(self):
@@ -185,8 +187,8 @@ class TestPlaylists(unittest.TestCase):
         ie = IviCompilationIE(dl)
         result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel/season2')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'dezhurnyi_angel/season2')
-        self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012) 2 сезон')
+        self.assertEqual(result['id'], 'dezhurnyi_angel/season2')
+        self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012) 2 сезон')
         self.assertTrue(len(result['entries']) >= 20)
         
     def test_imdb_list(self):
@@ -194,10 +196,20 @@ class TestPlaylists(unittest.TestCase):
         ie = ImdbListIE(dl)
         result = ie.extract('http://www.imdb.com/list/sMjedvGDd8U')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'sMjedvGDd8U')
-        self.assertEqual(result['title'], u'Animated and Family Films')
+        self.assertEqual(result['id'], 'sMjedvGDd8U')
+        self.assertEqual(result['title'], 'Animated and Family Films')
         self.assertTrue(len(result['entries']) >= 48)
 
+    def test_khanacademy_topic(self):
+        dl = FakeYDL()
+        ie = KhanAcademyIE(dl)
+        result = ie.extract('https://www.khanacademy.org/math/applied-math/cryptography')
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['id'], 'cryptography')
+        self.assertEqual(result['title'], 'Journey into cryptography')
+        self.assertEqual(result['description'], 'How have humans protected their secret messages through history? What has changed today?')
+        self.assertTrue(len(result['entries']) >= 3)
+
 
 if __name__ == '__main__':
     unittest.main()

youtube_dl/YoutubeDL.py

@@ -55,7 +55,7 @@ from .utils import (
 )
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
-from .PostProcessor import FFmpegMergerPP
+from .postprocessor import FFmpegMergerPP
 from .version import __version__
 
 
@@ -156,6 +156,10 @@ class YoutubeDL(object):
     the FileDownloader:
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
     noresizebuffer, retries, continuedl, noprogress, consoletitle
+
+    The following options are used by the post processors:
+    prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
+                       otherwise prefer avconv.
     """
 
     params = None
@@ -1062,7 +1066,9 @@ class YoutubeDL(object):
                 res += '%4dk ' % fdict['tbr']
             if (fdict.get('vcodec') is not None and
                     fdict.get('vcodec') != 'none'):
-                res += '%-5s@' % fdict['vcodec']
+                res += '%-5s' % fdict['vcodec']
+                if fdict.get('vbr') is not None:
+                    res += '@'
             elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
                 res += 'video@'
             if fdict.get('vbr') is not None:

youtube_dl/__init__.py

@@ -38,6 +38,7 @@ __authors__  = (
     'Takuya Tsuchida',
     'Sergey M.',
     'Michael Orlitzky',
+    'Chris Gahan',
 )
 
 __license__ = 'Public Domain'
@@ -74,11 +75,12 @@ from .FileDownloader import (
 from .extractor import gen_extractors
 from .version import __version__
 from .YoutubeDL import YoutubeDL
-from .PostProcessor import (
+from .postprocessor import (
     FFmpegMetadataPP,
     FFmpegVideoConvertor,
     FFmpegExtractAudioPP,
     FFmpegEmbedSubtitlePP,
+    XAttrMetadataPP,
 )
 
 
@@ -214,8 +216,12 @@ def parseOpts(overrideArguments=None):
     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
-    selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
-    selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
+    selection.add_option(
+        '--datebefore', metavar='DATE', dest='datebefore', default=None,
+        help='download only videos uploaded on or before this date (i.e. inclusive)')
+    selection.add_option(
+        '--dateafter', metavar='DATE', dest='dateafter', default=None,
+        help='download only videos uploaded on or after this date (i.e. inclusive)')
     selection.add_option(
         '--min-views', metavar='COUNT', dest='min_views',
         default=None, type=int,
@@ -240,7 +246,7 @@ def parseOpts(overrideArguments=None):
     authentication.add_option('-n', '--netrc',
             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
     authentication.add_option('--video-password',
-            dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)')
+            dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 
 
     video_format.add_option('-f', '--format',
@@ -253,7 +259,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option('--max-quality',
             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
     video_format.add_option('-F', '--list-formats',
-            action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
+            action='store_true', dest='listformats', help='list all available formats')
 
     subtitles.add_option('--write-sub', '--write-srt',
             action='store_true', dest='writesubtitles',
@@ -327,7 +333,7 @@ def parseOpts(overrideArguments=None):
             action='store_true', dest='verbose', help='print various debugging information', default=False)
     verbosity.add_option('--dump-intermediate-pages',
             action='store_true', dest='dump_intermediate_pages', default=False,
-            help='print downloaded pages to debug problems(very verbose)')
+            help='print downloaded pages to debug problems (very verbose)')
     verbosity.add_option('--write-pages',
             action='store_true', dest='write_pages', default=False,
             help='Write downloaded intermediary pages to files in the current directory to debug problems')
@@ -353,11 +359,11 @@ def parseOpts(overrideArguments=None):
                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
                   '%(autonumber)s to get an automatically incremented number, '
                   '%(ext)s for the filename extension, '
-                  '%(format)s for the format description (like "22 - 1280x720" or "HD"),'
-                  '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"),'
+                  '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
+                  '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
                   '%(upload_date)s for the upload date (YYYYMMDD), '
                   '%(extractor)s for the provider (youtube, metacafe, etc), '
-                  '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
+                  '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
                   'Use - to output to stdout. Can also be used to download to a different directory, '
                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
@@ -371,7 +377,7 @@ def parseOpts(overrideArguments=None):
             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
     filesystem.add_option('--load-info',
             dest='load_info_filename', metavar='FILE',
-            help='json file containing the video information (created with the "--write-json" option')
+            help='json file containing the video information (created with the "--write-json" option)')
     filesystem.add_option('-w', '--no-overwrites',
             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
     filesystem.add_option('-c', '--continue',
@@ -415,7 +421,13 @@ def parseOpts(overrideArguments=None):
     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
             help='embed subtitles in the video (only for mp4 videos)')
     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
-            help='add metadata to the files')
+            help='write metadata to the video file')
+    postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
+            help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
+    postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
+        help='Prefer avconv over ffmpeg for running the postprocessors (default)')
+    postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
+        help='Prefer ffmpeg over avconv for running the postprocessors')
 
 
     parser.add_option_group(general)
@@ -523,7 +535,7 @@ def _real_main(argv=None):
     all_urls = batchurls + args
     all_urls = [url.strip() for url in all_urls]
     _enc = preferredencoding()
-    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url]
+    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 
     extractors = gen_extractors()
 
@@ -553,7 +565,7 @@ def _real_main(argv=None):
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
         parser.error(u'using .netrc conflicts with giving username/password')
     if opts.password is not None and opts.username is None:
-        parser.error(u' account username missing\n')
+        parser.error(u'account username missing\n')
     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
         parser.error(u'using output template conflicts with using title, video ID or auto number')
     if opts.usetitle and opts.useid:
@@ -627,6 +639,7 @@ def _real_main(argv=None):
                      u' template'.format(outtmpl))
 
     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
+    download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
@@ -694,13 +707,14 @@ def _real_main(argv=None):
         'cachedir': opts.cachedir,
         'youtube_print_sig_code': opts.youtube_print_sig_code,
         'age_limit': opts.age_limit,
-        'download_archive': opts.download_archive,
+        'download_archive': download_archive_fn,
         'cookiefile': opts.cookiefile,
         'nocheckcertificate': opts.no_check_certificate,
         'proxy': opts.proxy,
         'socket_timeout': opts.socket_timeout,
         'bidi_workaround': opts.bidi_workaround,
         'debug_printtraffic': opts.debug_printtraffic,
+        'prefer_ffmpeg': opts.prefer_ffmpeg,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
@@ -717,6 +731,8 @@ def _real_main(argv=None):
             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
         if opts.embedsubtitles:
             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
+        if opts.xattrs:
+            ydl.add_post_processor(XAttrMetadataPP())
 
         # Update version
         if opts.update_self:

youtube_dl/downloader/hls.py

@@ -29,7 +29,7 @@ class HlsFD(FileDownloader):
         retval = subprocess.call(cmd)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))
+            self.to_screen(u'\r[%s] %s bytes' % (cmd[0], fsize))
             self.try_rename(tmpfilename, filename)
             self._hook_progress({
                 'downloaded_bytes': fsize,

youtube_dl/extractor/__init__.py

@@ -65,7 +65,8 @@ from .francetv import (
     PluzzIE,
     FranceTvInfoIE,
     FranceTVIE,
-    GenerationQuoiIE
+    GenerationQuoiIE,
+    CultureboxIE,
 )
 from .freesound import FreesoundIE
 from .funnyordie import FunnyOrDieIE
@@ -98,6 +99,7 @@ from .justintv import JustinTVIE
 from .jpopsukitv import JpopsukiIE
 from .kankan import KankanIE
 from .keezmovies import KeezMoviesIE
+from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
 from .keek import KeekIE
 from .liveleak import LiveLeakIE
@@ -112,6 +114,7 @@ from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mit import TechTVMITIE, MITIE
 from .mixcloud import MixcloudIE
+from .mpora import MporaIE
 from .mofosex import MofosexIE
 from .mtv import MTVIE
 from .muzu import MuzuTVIE
@@ -126,6 +129,7 @@ from .newgrounds import NewgroundsIE
 from .nhl import NHLIE, NHLVideocenterIE
 from .niconico import NiconicoIE
 from .ninegag import NineGagIE
+from .novamov import NovamovIE
 from .nowvideo import NowVideoIE
 from .ooyala import OoyalaIE
 from .orf import ORFIE
@@ -199,6 +203,7 @@ from .vimeo import (
     VimeoUserIE,
     VimeoAlbumIE,
     VimeoGroupsIE,
+    VimeoReviewIE,
 )
 from .vine import VineIE
 from .viki import VikiIE

youtube_dl/extractor/academicearth.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
@@ -5,7 +6,7 @@ from .common import InfoExtractor
 
 class AcademicEarthCourseIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/(?:courses|playlists)/(?P<id>[^?#/]+)'
-    IE_NAME = u'AcademicEarth:Course'
+    IE_NAME = 'AcademicEarth:Course'
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)

youtube_dl/extractor/appletrailers.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json
 
@@ -11,46 +13,46 @@ from ..utils import (
 class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TEST = {
-        u"url": u"http://trailers.apple.com/trailers/wb/manofsteel/",
-        u"playlist": [
+        "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
+        "playlist": [
             {
-                u"file": u"manofsteel-trailer4.mov",
-                u"md5": u"d97a8e575432dbcb81b7c3acb741f8a8",
-                u"info_dict": {
-                    u"duration": 111,
-                    u"title": u"Trailer 4",
-                    u"upload_date": u"20130523",
-                    u"uploader_id": u"wb",
+                "file": "manofsteel-trailer4.mov",
+                "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
+                "info_dict": {
+                    "duration": 111,
+                    "title": "Trailer 4",
+                    "upload_date": "20130523",
+                    "uploader_id": "wb",
                 },
             },
             {
-                u"file": u"manofsteel-trailer3.mov",
-                u"md5": u"b8017b7131b721fb4e8d6f49e1df908c",
-                u"info_dict": {
-                    u"duration": 182,
-                    u"title": u"Trailer 3",
-                    u"upload_date": u"20130417",
-                    u"uploader_id": u"wb",
+                "file": "manofsteel-trailer3.mov",
+                "md5": "b8017b7131b721fb4e8d6f49e1df908c",
+                "info_dict": {
+                    "duration": 182,
+                    "title": "Trailer 3",
+                    "upload_date": "20130417",
+                    "uploader_id": "wb",
                 },
             },
             {
-                u"file": u"manofsteel-trailer.mov",
-                u"md5": u"d0f1e1150989b9924679b441f3404d48",
-                u"info_dict": {
-                    u"duration": 148,
-                    u"title": u"Trailer",
-                    u"upload_date": u"20121212",
-                    u"uploader_id": u"wb",
+                "file": "manofsteel-trailer.mov",
+                "md5": "d0f1e1150989b9924679b441f3404d48",
+                "info_dict": {
+                    "duration": 148,
+                    "title": "Trailer",
+                    "upload_date": "20121212",
+                    "uploader_id": "wb",
                 },
             },
             {
-                u"file": u"manofsteel-teaser.mov",
-                u"md5": u"5fe08795b943eb2e757fa95cb6def1cb",
-                u"info_dict": {
-                    u"duration": 93,
-                    u"title": u"Teaser",
-                    u"upload_date": u"20120721",
-                    u"uploader_id": u"wb",
+                "file": "manofsteel-teaser.mov",
+                "md5": "5fe08795b943eb2e757fa95cb6def1cb",
+                "info_dict": {
+                    "duration": 93,
+                    "title": "Teaser",
+                    "upload_date": "20120721",
+                    "uploader_id": "wb",
                 },
             }
         ]

youtube_dl/extractor/archiveorg.py

@@ -1,9 +1,10 @@
+from __future__ import unicode_literals
+
 import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     unified_strdate,
 )
 
@@ -13,23 +14,22 @@ class ArchiveOrgIE(InfoExtractor):
     IE_DESC = 'archive.org videos'
     _VALID_URL = r'(?:https?://)?(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$'
     _TEST = {
-        u"url": u"http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect",
-        u'file': u'XD300-23_68HighlightsAResearchCntAugHumanIntellect.ogv',
-        u'md5': u'8af1d4cf447933ed3c7f4871162602db',
-        u'info_dict': {
-            u"title": u"1968 Demo - FJCC Conference Presentation Reel #1",
-            u"description": u"Reel 1 of 3: Also known as the \"Mother of All Demos\", Doug Engelbart's presentation at the Fall Joint Computer Conference in San Francisco, December 9, 1968 titled \"A Research Center for Augmenting Human Intellect.\" For this presentation, Doug and his team astonished the audience by not only relating their research, but demonstrating it live. This was the debut of the mouse, interactive computing, hypermedia, computer supported software engineering, video teleconferencing, etc. See also <a href=\"http://dougengelbart.org/firsts/dougs-1968-demo.html\" rel=\"nofollow\">Doug's 1968 Demo page</a> for more background, highlights, links, and the detailed paper published in this conference proceedings. Filmed on 3 reels: Reel 1 | <a href=\"http://www.archive.org/details/XD300-24_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 2</a> | <a href=\"http://www.archive.org/details/XD300-25_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 3</a>",
-            u"upload_date": u"19681210",
-            u"uploader": u"SRI International"
+        "url": "http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect",
+        'file': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.ogv',
+        'md5': '8af1d4cf447933ed3c7f4871162602db',
+        'info_dict': {
+            "title": "1968 Demo - FJCC Conference Presentation Reel #1",
+            "description": "Reel 1 of 3: Also known as the \"Mother of All Demos\", Doug Engelbart's presentation at the Fall Joint Computer Conference in San Francisco, December 9, 1968 titled \"A Research Center for Augmenting Human Intellect.\" For this presentation, Doug and his team astonished the audience by not only relating their research, but demonstrating it live. This was the debut of the mouse, interactive computing, hypermedia, computer supported software engineering, video teleconferencing, etc. See also <a href=\"http://dougengelbart.org/firsts/dougs-1968-demo.html\" rel=\"nofollow\">Doug's 1968 Demo page</a> for more background, highlights, links, and the detailed paper published in this conference proceedings. Filmed on 3 reels: Reel 1 | <a href=\"http://www.archive.org/details/XD300-24_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 2</a> | <a href=\"http://www.archive.org/details/XD300-25_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 3</a>",
+            "upload_date": "19681210",
+            "uploader": "SRI International"
         }
     }
 
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        json_url = url + (u'?' if u'?' in url else '&') + u'output=json'
+        json_url = url + ('?' if '?' in url else '&') + 'output=json'
         json_data = self._download_webpage(json_url, video_id)
         data = json.loads(json_data)
 
@@ -38,16 +38,16 @@ class ArchiveOrgIE(InfoExtractor):
         uploader = data['metadata']['creator'][0]
         upload_date = unified_strdate(data['metadata']['date'][0])
 
-        formats = [{
+        formats = [
+            {
                 'format': fdata['format'],
                 'url': 'http://' + data['server'] + data['dir'] + fn,
                 'file_size': int(fdata['size']),
             }
-            for fn,fdata in data['files'].items()
+            for fn, fdata in data['files'].items()
             if 'Video' in fdata['format']]
-        formats.sort(key=lambda fdata: fdata['file_size'])
-        for f in formats:
-            f['ext'] = determine_ext(f['url'])
+
+        self._sort_formats(formats)
 
         return {
             '_type': 'video',

youtube_dl/extractor/arte.py

@@ -1,4 +1,6 @@
 # encoding: utf-8
+from __future__ import unicode_literals
+
 import re
 import json
 
@@ -22,7 +24,7 @@ class ArteTvIE(InfoExtractor):
     _LIVEWEB_URL = r'(?:http://)?liveweb\.arte\.tv/(?P<lang>fr|de)/(?P<subpage>.+?)/(?P<name>.+)'
     _LIVE_URL = r'index-[0-9]+\.html$'
 
-    IE_NAME = u'arte.tv'
+    IE_NAME = 'arte.tv'
 
     @classmethod
     def suitable(cls, url):
@@ -37,7 +39,7 @@ class ArteTvIE(InfoExtractor):
     #         r'src="(.*?/videothek_js.*?\.js)',
     #         0,
     #         [
-    #             (1, 'url', u'Invalid URL: %s' % url)
+    #             (1, 'url', 'Invalid URL: %s' % url)
     #         ]
     #     )
     #     http_host = url.split('/')[2]
@@ -49,12 +51,12 @@ class ArteTvIE(InfoExtractor):
     #             '(rtmp://.*?)\'',
     #         re.DOTALL,
     #         [
-    #             (1, 'path',   u'could not extract video path: %s' % url),
-    #             (2, 'player', u'could not extract video player: %s' % url),
-    #             (3, 'url',    u'could not extract video url: %s' % url)
+    #             (1, 'path',   'could not extract video path: %s' % url),
+    #             (2, 'player', 'could not extract video player: %s' % url),
+    #             (3, 'url',    'could not extract video url: %s' % url)
     #         ]
     #     )
-    #     video_url = u'%s/%s' % (info.get('url'), info.get('path'))
+    #     video_url = '%s/%s' % (info.get('url'), info.get('path'))
 
     def _real_extract(self, url):
         mobj = re.match(self._VIDEOS_URL, url)
@@ -107,9 +109,9 @@ class ArteTvIE(InfoExtractor):
     def _extract_liveweb(self, url, name, lang):
         """Extract form http://liveweb.arte.tv/"""
         webpage = self._download_webpage(url, name)
-        video_id = self._search_regex(r'eventId=(\d+?)("|&)', webpage, u'event id')
+        video_id = self._search_regex(r'eventId=(\d+?)("|&)', webpage, 'event id')
         config_doc = self._download_xml('http://download.liveweb.arte.tv/o21/liveweb/events/event-%s.xml' % video_id,
-                                            video_id, u'Downloading information')
+                                            video_id, 'Downloading information')
         event_doc = config_doc.find('event')
         url_node = event_doc.find('video').find('urlHd')
         if url_node is None:
@@ -124,7 +126,7 @@ class ArteTvIE(InfoExtractor):
 
 
 class ArteTVPlus7IE(InfoExtractor):
-    IE_NAME = u'arte.tv:+7'
+    IE_NAME = 'arte.tv:+7'
     _VALID_URL = r'https?://www\.arte.tv/guide/(?P<lang>fr|de)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
 
     @classmethod
@@ -207,7 +209,7 @@ class ArteTVPlus7IE(InfoExtractor):
             if bitrate is not None:
                 quality += '-%d' % bitrate
             if format_info.get('versionCode') is not None:
-                format_id = u'%s-%s' % (quality, format_info['versionCode'])
+                format_id = '%s-%s' % (quality, format_info['versionCode'])
             else:
                 format_id = quality
             info = {
@@ -216,7 +218,7 @@ class ArteTVPlus7IE(InfoExtractor):
                 'width': format_info.get('width'),
                 'height': height,
             }
-            if format_info['mediaType'] == u'rtmp':
+            if format_info['mediaType'] == 'rtmp':
                 info['url'] = format_info['streamer']
                 info['play_path'] = 'mp4:' + format_info['url']
                 info['ext'] = 'flv'
@@ -231,27 +233,27 @@ class ArteTVPlus7IE(InfoExtractor):
 
 # It also uses the arte_vp_url url from the webpage to extract the information
 class ArteTVCreativeIE(ArteTVPlus7IE):
-    IE_NAME = u'arte.tv:creative'
+    IE_NAME = 'arte.tv:creative'
     _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/magazine?/(?P<id>.+)'
 
     _TEST = {
-        u'url': u'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
-        u'file': u'050489-002.mp4',
-        u'info_dict': {
-            u'title': u'Agentur Amateur / Agence Amateur #2 : Corporate Design',
+        'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
+        'file': '050489-002.mp4',
+        'info_dict': {
+            'title': 'Agentur Amateur / Agence Amateur #2 : Corporate Design',
         },
     }
 
 
 class ArteTVFutureIE(ArteTVPlus7IE):
-    IE_NAME = u'arte.tv:future'
+    IE_NAME = 'arte.tv:future'
     _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(thema|sujet)/.*?#article-anchor-(?P<id>\d+)'
 
     _TEST = {
-        u'url': u'http://future.arte.tv/fr/sujet/info-sciences#article-anchor-7081',
-        u'file': u'050940-003.mp4',
-        u'info_dict': {
-            u'title': u'Les champignons au secours de la planète',
+        'url': 'http://future.arte.tv/fr/sujet/info-sciences#article-anchor-7081',
+        'file': '050940-003.mp4',
+        'info_dict': {
+            'title': 'Les champignons au secours de la planète',
         },
     }
 
@@ -263,7 +265,7 @@ class ArteTVFutureIE(ArteTVPlus7IE):
 
 
 class ArteTVDDCIE(ArteTVPlus7IE):
-    IE_NAME = u'arte.tv:ddc'
+    IE_NAME = 'arte.tv:ddc'
     _VALID_URL = r'http?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>.+)'
 
     def _real_extract(self, url):

youtube_dl/extractor/auengine.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -7,13 +9,14 @@ from ..utils import (
     ExtractorError,
 )
 
+
 class AUEngineIE(InfoExtractor):
     _TEST = {
-        u'url': u'http://auengine.com/embed.php?file=lfvlytY6&w=650&h=370',
-        u'file': u'lfvlytY6.mp4',
-        u'md5': u'48972bdbcf1a3a2f5533e62425b41d4f',
-        u'info_dict': {
-            u"title": u"[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]"
+        'url': 'http://auengine.com/embed.php?file=lfvlytY6&w=650&h=370',
+        'file': 'lfvlytY6.mp4',
+        'md5': '48972bdbcf1a3a2f5533e62425b41d4f',
+        'info_dict': {
+            'title': '[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]'
         }
     }
     _VALID_URL = r'(?:http://)?(?:www\.)?auengine\.com/embed\.php\?.*?file=([^&]+).*?'
@@ -23,7 +26,7 @@ class AUEngineIE(InfoExtractor):
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<title>(?P<title>.+?)</title>',
-                webpage, u'title')
+                webpage, 'title')
         title = title.strip()
         links = re.findall(r'\s(?:file|url):\s*["\']([^\'"]+)["\']', webpage)
         links = map(compat_urllib_parse.unquote, links)
@@ -37,7 +40,7 @@ class AUEngineIE(InfoExtractor):
                 video_url = link
         if not video_url:
             raise ExtractorError(u'Could not find video URL')
-        ext = u'.' + determine_ext(video_url)
+        ext = '.' + determine_ext(video_url)
         if ext == title[-len(ext):]:
             title = title[:-len(ext)]
 

youtube_dl/extractor/bambuser.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json
 import itertools
@@ -9,26 +11,26 @@ from ..utils import (
 
 
 class BambuserIE(InfoExtractor):
-    IE_NAME = u'bambuser'
+    IE_NAME = 'bambuser'
     _VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)'
     _API_KEY = '005f64509e19a868399060af746a00aa'
 
     _TEST = {
-        u'url': u'http://bambuser.com/v/4050584',
+        'url': 'http://bambuser.com/v/4050584',
         # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
-        #u'md5': u'fba8f7693e48fd4e8641b3fd5539a641',
-        u'info_dict': {
-            u'id': u'4050584',
-            u'ext': u'flv',
-            u'title': u'Education engineering days - lightning talks',
-            u'duration': 3741,
-            u'uploader': u'pixelversity',
-            u'uploader_id': u'344706',
+        #u'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
+        'info_dict': {
+            'id': '4050584',
+            'ext': 'flv',
+            'title': 'Education engineering days - lightning talks',
+            'duration': 3741,
+            'uploader': 'pixelversity',
+            'uploader_id': '344706',
         },
-        u'params': {
+        'params': {
             # It doesn't respect the 'Range' header, it would download the whole video
             # caused the travis builds to fail: https://travis-ci.org/rg3/youtube-dl/jobs/14493845#L59
-            u'skip_download': True,
+            'skip_download': True,
         },
     }
 
@@ -53,7 +55,7 @@ class BambuserIE(InfoExtractor):
 
 
 class BambuserChannelIE(InfoExtractor):
-    IE_NAME = u'bambuser:channel'
+    IE_NAME = 'bambuser:channel'
     _VALID_URL = r'https?://bambuser\.com/channel/(?P<user>.*?)(?:/|#|\?|$)'
     # The maximum number we can get with each request
     _STEP = 50
@@ -72,7 +74,7 @@ class BambuserChannelIE(InfoExtractor):
             # Without setting this header, we wouldn't get any result
             req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
             info_json = self._download_webpage(req, user,
-                u'Downloading page %d' % i)
+                'Downloading page %d' % i)
             results = json.loads(info_json)['result']
             if len(results) == 0:
                 break

youtube_dl/extractor/bandcamp.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import json
 import re
 
@@ -12,14 +14,14 @@ from ..utils import (
 class BandcampIE(InfoExtractor):
     _VALID_URL = r'http://.*?\.bandcamp\.com/track/(?P<title>.*)'
     _TESTS = [{
-        u'url': u'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
-        u'file': u'1812978515.mp3',
-        u'md5': u'c557841d5e50261777a6585648adf439',
-        u'info_dict': {
-            u"title": u"youtube-dl  \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad",
-            u"duration": 10,
+        'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
+        'file': '1812978515.mp3',
+        'md5': 'c557841d5e50261777a6585648adf439',
+        'info_dict': {
+            "title": "youtube-dl  \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad",
+            "duration": 10,
         },
-        u'skip': u'There is a limit of 200 free downloads / month for the test song'
+        '_skip': 'There is a limit of 200 free downloads / month for the test song'
     }]
 
     def _real_extract(self, url):
@@ -58,7 +60,7 @@ class BandcampIE(InfoExtractor):
                     'duration': duration,
                 }
             else:
-                raise ExtractorError(u'No free songs found')
+                raise ExtractorError('No free songs found')
 
         download_link = m_download.group(1)
         video_id = re.search(
@@ -72,56 +74,56 @@ class BandcampIE(InfoExtractor):
                          download_webpage, re.MULTILINE).group(1)
         info = json.loads(info)[0]
         # We pick mp3-320 for now, until format selection can be easily implemented.
-        mp3_info = info[u'downloads'][u'mp3-320']
+        mp3_info = info['downloads']['mp3-320']
         # If we try to use this url it says the link has expired
-        initial_url = mp3_info[u'url']
+        initial_url = mp3_info['url']
         re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$'
         m_url = re.match(re_url, initial_url)
         #We build the url we will use to get the final track url
         # This url is build in Bandcamp in the script download_bunde_*.js
         request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts'))
-        final_url_webpage = self._download_webpage(request_url, id, 'Requesting download url')
+        final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
         # If we could correctly generate the .rand field the url would be
         #in the "download_url" key
         final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1)
 
         return {
             'id': video_id,
-            'title': info[u'title'],
+            'title': info['title'],
             'ext': 'mp3',
             'vcodec': 'none',
             'url': final_url,
-            'thumbnail': info[u'thumb_url'],
-            'uploader': info[u'artist'],
+            'thumbnail': info.get('thumb_url'),
+            'uploader': info.get('artist'),
         }
 
 
 class BandcampAlbumIE(InfoExtractor):
-    IE_NAME = u'Bandcamp:album'
+    IE_NAME = 'Bandcamp:album'
     _VALID_URL = r'http://.*?\.bandcamp\.com/album/(?P<title>.*)'
 
     _TEST = {
-        u'url': u'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
-        u'playlist': [
+        'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
+        'playlist': [
             {
-                u'file': u'1353101989.mp3',
-                u'md5': u'39bc1eded3476e927c724321ddf116cf',
-                u'info_dict': {
-                    u'title': u'Intro',
+                'file': '1353101989.mp3',
+                'md5': '39bc1eded3476e927c724321ddf116cf',
+                'info_dict': {
+                    'title': 'Intro',
                 }
             },
             {
-                u'file': u'38097443.mp3',
-                u'md5': u'1a2c32e2691474643e912cc6cd4bffaa',
-                u'info_dict': {
-                    u'title': u'Kero One - Keep It Alive (Blazo remix)',
+                'file': '38097443.mp3',
+                'md5': '1a2c32e2691474643e912cc6cd4bffaa',
+                'info_dict': {
+                    'title': 'Kero One - Keep It Alive (Blazo remix)',
                 }
             },
         ],
-        u'params': {
-            u'playlistend': 2
+        'params': {
+            'playlistend': 2
         },
-        u'skip': u'Bancamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test'
+        'skip': 'Bancamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test'
     }
 
     def _real_extract(self, url):
@@ -130,11 +132,11 @@ class BandcampAlbumIE(InfoExtractor):
         webpage = self._download_webpage(url, title)
         tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage)
         if not tracks_paths:
-            raise ExtractorError(u'The page doesn\'t contain any tracks')
+            raise ExtractorError('The page doesn\'t contain any tracks')
         entries = [
             self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
             for t_path in tracks_paths]
-        title = self._search_regex(r'album_title : "(.*?)"', webpage, u'title')
+        title = self._search_regex(r'album_title : "(.*?)"', webpage, 'title')
         return {
             '_type': 'playlist',
             'title': title,

youtube_dl/extractor/blinkx.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import datetime
 import json
 import re
@@ -10,19 +12,19 @@ from ..utils import (
 
 class BlinkxIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
-    _IE_NAME = u'blinkx'
+    IE_NAME = 'blinkx'
 
     _TEST = {
-        u'url': u'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB',
-        u'file': u'8aQUy7GV.mp4',
-        u'md5': u'2e9a07364af40163a908edbf10bb2492',
-        u'info_dict': {
-            u"title": u"Police Car Rolls Away",
-            u"uploader": u"stupidvideos.com",
-            u"upload_date": u"20131215",
-            u"description": u"A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!",
-            u"duration": 14.886,
-            u"thumbnails": [{
+        'url': 'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB',
+        'file': '8aQUy7GV.mp4',
+        'md5': '2e9a07364af40163a908edbf10bb2492',
+        'info_dict': {
+            "title": "Police Car Rolls Away",
+            "uploader": "stupidvideos.com",
+            "upload_date": "20131215",
+            "description": "A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!",
+            "duration": 14.886,
+            "thumbnails": [{
                 "width": 100,
                 "height": 76,
                 "url": "http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg",
@@ -30,17 +32,17 @@ class BlinkxIE(InfoExtractor):
         },
     }
 
-    def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
+    def _real_extract(self, rl):
+        m = re.match(self._VALID_URL, rl)
         video_id = m.group('id')
         display_id = video_id[:8]
 
         api_url = (u'https://apib4.blinkx.com/api.php?action=play_video&' +
-                   u'video=%s' % video_id)
+                   'video=%s' % video_id)
         data_json = self._download_webpage(api_url, display_id)
         data = json.loads(data_json)['api']['results'][0]
         dt = datetime.datetime.fromtimestamp(data['pubdate_epoch'])
-        upload_date = dt.strftime('%Y%m%d')
+        pload_date = dt.strftime('%Y%m%d')
 
         duration = None
         thumbnails = []
@@ -86,7 +88,7 @@ class BlinkxIE(InfoExtractor):
             'title': data['title'],
             'formats': formats,
             'uploader': data['channel_name'],
-            'upload_date': upload_date,
+            'upload_date': pload_date,
             'description': data.get('description'),
             'thumbnails': thumbnails,
             'duration': duration,

youtube_dl/extractor/bloomberg.py

@@ -1,6 +1,7 @@
 import re
 
 from .common import InfoExtractor
+from .ooyala import OoyalaIE
 
 
 class BloombergIE(InfoExtractor):
@@ -23,5 +24,5 @@ class BloombergIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
         webpage = self._download_webpage(url, name)
-        ooyala_url = self._og_search_video_url(webpage)
-        return self.url_result(ooyala_url, ie='Ooyala')
+        ooyala_code = self._search_regex(r'<source src="http://player.ooyala.com/player/[^/]+/([^".]+)', webpage, u'ooyala url')
+        return OoyalaIE._build_url_result(ooyala_code)

youtube_dl/extractor/brightcove.py

@@ -1,4 +1,5 @@
 # encoding: utf-8
+from __future__ import unicode_literals
 
 import re
 import json
@@ -13,6 +14,7 @@ from ..utils import (
     compat_urllib_request,
 
     ExtractorError,
+    unsmuggle_url,
 )
 
 
@@ -24,47 +26,47 @@ class BrightcoveIE(InfoExtractor):
     _TESTS = [
         {
             # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
-            u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
-            u'file': u'2371591881001.mp4',
-            u'md5': u'5423e113865d26e40624dce2e4b45d95',
-            u'note': u'Test Brightcove downloads and detection in GenericIE',
-            u'info_dict': {
-                u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
-                u'uploader': u'8TV',
-                u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
+            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
+            'file': '2371591881001.mp4',
+            'md5': '5423e113865d26e40624dce2e4b45d95',
+            'note': 'Test Brightcove downloads and detection in GenericIE',
+            'info_dict': {
+                'title': 'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
+                'uploader': '8TV',
+                'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
             }
         },
         {
             # From http://medianetwork.oracle.com/video/player/1785452137001
-            u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
-            u'file': u'1785452137001.flv',
-            u'info_dict': {
-                u'title': u'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
-                u'description': u'John Rose speaks at the JVM Language Summit, August 1, 2012.',
-                u'uploader': u'Oracle',
+            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
+            'file': '1785452137001.flv',
+            'info_dict': {
+                'title': 'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
+                'description': 'John Rose speaks at the JVM Language Summit, August 1, 2012.',
+                'uploader': 'Oracle',
             },
         },
         {
             # From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
-            u'url': u'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
-            u'info_dict': {
-                u'id': u'2750934548001',
-                u'ext': u'mp4',
-                u'title': u'This Bracelet Acts as a Personal Thermostat',
-                u'description': u'md5:547b78c64f4112766ccf4e151c20b6a0',
-                u'uploader': u'Mashable',
+            'url': 'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
+            'info_dict': {
+                'id': '2750934548001',
+                'ext': 'mp4',
+                'title': 'This Bracelet Acts as a Personal Thermostat',
+                'description': 'md5:547b78c64f4112766ccf4e151c20b6a0',
+                'uploader': 'Mashable',
             },
         },
         {
             # test that the default referer works
             # from http://national.ballet.ca/interact/video/Lost_in_Motion_II/
-            u'url': u'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
-            u'info_dict': {
-                u'id': u'2878862109001',
-                u'ext': u'mp4',
-                u'title': u'Lost in Motion II',
-                u'description': u'md5:363109c02998fee92ec02211bd8000df',
-                u'uploader': u'National Ballet of Canada',
+            'url': 'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
+            'info_dict': {
+                'id': '2878862109001',
+                'ext': 'mp4',
+                'title': 'Lost in Motion II',
+                'description': 'md5:363109c02998fee92ec02211bd8000df',
+                'uploader': 'National Ballet of Canada',
             },
         },
     ]
@@ -80,13 +82,13 @@ class BrightcoveIE(InfoExtractor):
         object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
                             lambda m: m.group(1) + '/>', object_str)
         # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
-        object_str = object_str.replace(u'<--', u'<!--')
+        object_str = object_str.replace('<--', '<!--')
 
         object_doc = xml.etree.ElementTree.fromstring(object_str)
-        assert u'BrightcoveExperience' in object_doc.attrib['class']
-        params = {'flashID': object_doc.attrib['id'],
-                  'playerID': find_xpath_attr(object_doc, './param', 'name', 'playerID').attrib['value'],
-                  }
+        assert 'BrightcoveExperience' in object_doc.attrib['class']
+        params = {
+            'playerID': find_xpath_attr(object_doc, './param', 'name', 'playerID').attrib['value'],
+        }
         def find_param(name):
             node = find_xpath_attr(object_doc, './param', 'name', name)
             if node is not None:
@@ -120,6 +122,8 @@ class BrightcoveIE(InfoExtractor):
             return None
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         # Change the 'videoId' and others field to '@videoPlayer'
         url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url)
         # Change bckey (used by bcove.me urls) to playerKey
@@ -130,9 +134,10 @@ class BrightcoveIE(InfoExtractor):
 
         videoPlayer = query.get('@videoPlayer')
         if videoPlayer:
-            return self._get_video_info(videoPlayer[0], query_str, query,
-                # We set the original url as the default 'Referer' header
-                referer=url)
+            # We set the original url as the default 'Referer' header
+            referer = smuggled_data.get('Referer', url)
+            return self._get_video_info(
+                videoPlayer[0], query_str, query, referer=referer)
         else:
             player_key = query['playerKey']
             return self._get_playlist_info(player_key[0])
@@ -156,11 +161,11 @@ class BrightcoveIE(InfoExtractor):
 
     def _get_playlist_info(self, player_key):
         playlist_info = self._download_webpage(self._PLAYLIST_URL_TEMPLATE % player_key,
-                                               player_key, u'Downloading playlist information')
+                                               player_key, 'Downloading playlist information')
 
         json_data = json.loads(playlist_info)
         if 'videoList' not in json_data:
-            raise ExtractorError(u'Empty playlist')
+            raise ExtractorError('Empty playlist')
         playlist_info = json_data['videoList']
         videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]
 
@@ -189,5 +194,5 @@ class BrightcoveIE(InfoExtractor):
                 'url': video_info['FLVFullLengthURL'],
             })
         else:
-            raise ExtractorError(u'Unable to extract video url for %s' % info['id'])
+            raise ExtractorError('Unable to extract video url for %s' % info['id'])
         return info

youtube_dl/extractor/c56.py

@@ -1,21 +1,21 @@
 # coding: utf-8
+from __future__ import unicode_literals
 
 import re
 import json
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+
 
 class C56IE(InfoExtractor):
     _VALID_URL = r'https?://((www|player)\.)?56\.com/(.+?/)?(v_|(play_album.+-))(?P<textid>.+?)\.(html|swf)'
-    IE_NAME = u'56.com'
-
-    _TEST ={
-        u'url': u'http://www.56.com/u39/v_OTM0NDA3MTY.html',
-        u'file': u'93440716.flv',
-        u'md5': u'e59995ac63d0457783ea05f93f12a866',
-        u'info_dict': {
-            u'title': u'网事知多少 第32期：车怒',
+    IE_NAME = '56.com'
+    _TEST = {
+        'url': 'http://www.56.com/u39/v_OTM0NDA3MTY.html',
+        'file': '93440716.flv',
+        'md5': 'e59995ac63d0457783ea05f93f12a866',
+        'info_dict': {
+            'title': '网事知多少 第32期：车怒',
         },
     }
 
@@ -23,14 +23,18 @@ class C56IE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
         text_id = mobj.group('textid')
         info_page = self._download_webpage('http://vxml.56.com/json/%s/' % text_id,
-                                           text_id, u'Downloading video info')
+                                           text_id, 'Downloading video info')
         info = json.loads(info_page)['info']
-        best_format = sorted(info['rfiles'], key=lambda f: int(f['filesize']))[-1]
-        video_url = best_format['url']
+        formats = [{
+            'format_id': f['type'],
+            'filesize': int(f['filesize']),
+            'url': f['url']
+        } for f in info['rfiles']]
+        self._sort_formats(formats)
 
-        return {'id': info['vid'],
-                'title': info['Subject'],
-                'url': video_url,
-                'ext': determine_ext(video_url),
-                'thumbnail': info.get('bimg') or info.get('img'),
-                }
+        return {
+            'id': info['vid'],
+            'title': info['Subject'],
+            'formats': formats,
+            'thumbnail': info.get('bimg') or info.get('img'),
+        }

youtube_dl/extractor/cnn.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -12,14 +14,14 @@ class CNNIE(InfoExtractor):
         (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn|(?=&)))'''
 
     _TESTS = [{
-        u'url': u'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
-        u'file': u'sports_2013_06_09_nadal-1-on-1.cnn.mp4',
-        u'md5': u'3e6121ea48df7e2259fe73a0628605c4',
-        u'info_dict': {
-            u'title': u'Nadal wins 8th French Open title',
-            u'description': u'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
-            u'duration': 135,
-            u'upload_date': u'20130609',
+        'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
+        'file': 'sports_2013_06_09_nadal-1-on-1.cnn.mp4',
+        'md5': '3e6121ea48df7e2259fe73a0628605c4',
+        'info_dict': {
+            'title': 'Nadal wins 8th French Open title',
+            'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
+            'duration': 135,
+            'upload_date': '20130609',
         },
     },
     {
@@ -36,7 +38,7 @@ class CNNIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         path = mobj.group('path')
         page_title = mobj.group('title')
-        info_url = u'http://cnn.com/video/data/3.0/%s/index.xml' % path
+        info_url = 'http://cnn.com/video/data/3.0/%s/index.xml' % path
         info = self._download_xml(info_url, page_title)
 
         formats = []

youtube_dl/extractor/common.py

@@ -1,4 +1,5 @@
 import base64
+import json
 import os
 import re
 import socket
@@ -73,6 +74,10 @@ class InfoExtractor(object):
                                  by this field.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
+                    * quality    Order number of the video quality of this
+                                 format, irrespective of the file format.
+                                 -1 for default (order by other properties),
+                                 -2 or smaller for less than default.
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)
@@ -256,6 +261,15 @@ class InfoExtractor(object):
             xml_string = transform_source(xml_string)
         return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
 
+    def _download_json(self, url_or_request, video_id,
+                       note=u'Downloading JSON metadata',
+                       errnote=u'Unable to download JSON metadata'):
+        json_string = self._download_webpage(url_or_request, video_id, note, errnote)
+        try:
+            return json.loads(json_string)
+        except ValueError as ve:
+            raise ExtractorError('Failed to download JSON', cause=ve)
+
     def report_warning(self, msg, video_id=None):
         idstr = u'' if video_id is None else u'%s: ' % video_id
         self._downloader.report_warning(
@@ -483,9 +497,11 @@ class InfoExtractor(object):
 
             return (
                 preference,
+                f.get('quality') if f.get('quality') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
                 ext_preference,
+                f.get('tbr') if f.get('tbr') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,

youtube_dl/extractor/defense.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json
 
@@ -5,15 +7,14 @@ from .common import InfoExtractor
 
 
 class DefenseGouvFrIE(InfoExtractor):
-    _IE_NAME = 'defense.gouv.fr'
+    IE_NAME = 'defense.gouv.fr'
     _VALID_URL = (r'http://.*?\.defense\.gouv\.fr/layout/set/'
         r'ligthboxvideo/base-de-medias/webtv/(.*)')
 
     _TEST = {
-        u'url': (u'http://www.defense.gouv.fr/layout/set/ligthboxvideo/'
-        u'base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1'),
-        u'file': u'11213.mp4',
-        u'md5': u'75bba6124da7e63d2d60b5244ec9430c',
+        'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
+        'file': '11213.mp4',
+        'md5': '75bba6124da7e63d2d60b5244ec9430c',
         "info_dict": {
             "title": "attaque-chimique-syrienne-du-21-aout-2013-1"
         }

youtube_dl/extractor/francetv.py

@@ -191,3 +191,29 @@ class GenerationQuoiIE(InfoExtractor):
         info = json.loads(info_json)
         return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
             ie='Dailymotion')
+
+
+class CultureboxIE(FranceTVBaseInfoExtractor):
+    IE_NAME = u'culturebox.francetvinfo.fr'
+    _VALID_URL = r'https?://culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)'
+
+    _TEST = {
+        u'url': u'http://culturebox.francetvinfo.fr/einstein-on-the-beach-au-theatre-du-chatelet-146813',
+        u'info_dict': {
+            u'id': u'EV_6785',
+            u'ext': u'mp4',
+            u'title': u'Einstein on the beach au Théâtre du Châtelet',
+            u'description': u'md5:9ce2888b1efefc617b5e58b3f6200eeb',
+        },
+        u'params': {
+            # m3u8 download
+            u'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        name = mobj.group('name')
+        webpage = self._download_webpage(url, name)
+        video_id = self._search_regex(r'"http://videos\.francetv\.fr/video/(.*?)"', webpage, u'video id')
+        return self._extract_video(video_id)

youtube_dl/extractor/generic.py

@@ -1,9 +1,12 @@
 # encoding: utf-8
 
+from __future__ import unicode_literals
+
 import os
 import re
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
     compat_urllib_error,
     compat_urllib_parse,
@@ -22,78 +25,78 @@ from .ooyala import OoyalaIE
 
 
 class GenericIE(InfoExtractor):
-    IE_DESC = u'Generic downloader that works on some sites'
+    IE_DESC = 'Generic downloader that works on some sites'
     _VALID_URL = r'.*'
-    IE_NAME = u'generic'
+    IE_NAME = 'generic'
     _TESTS = [
         {
-            u'url': u'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
-            u'file': u'13601338388002.mp4',
-            u'md5': u'6e15c93721d7ec9e9ca3fdbf07982cfd',
-            u'info_dict': {
-                u"uploader": u"www.hodiho.fr",
-                u"title": u"R\u00e9gis plante sa Jeep"
+            'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
+            'file': '13601338388002.mp4',
+            'md5': '6e15c93721d7ec9e9ca3fdbf07982cfd',
+            'info_dict': {
+                'uploader': 'www.hodiho.fr',
+                'title': 'R\u00e9gis plante sa Jeep',
             }
         },
         # embedded vimeo video
         {
-            u'add_ie': ['Vimeo'],
-            u'url': u'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
-            u'file': u'22444065.mp4',
-            u'md5': u'2903896e23df39722c33f015af0666e2',
-            u'info_dict': {
-                u'title': u'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
-                u"uploader_id": u"skillsmatter",
-                u"uploader": u"Skills Matter",
+            'add_ie': ['Vimeo'],
+            'url': 'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
+            'file': '22444065.mp4',
+            'md5': '2903896e23df39722c33f015af0666e2',
+            'info_dict': {
+                'title': 'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
+                'uploader_id': 'skillsmatter',
+                'uploader': 'Skills Matter',
             }
         },
         # bandcamp page with custom domain
         {
-            u'add_ie': ['Bandcamp'],
-            u'url': u'http://bronyrock.com/track/the-pony-mash',
-            u'file': u'3235767654.mp3',
-            u'info_dict': {
-                u'title': u'The Pony Mash',
-                u'uploader': u'M_Pallante',
+            'add_ie': ['Bandcamp'],
+            'url': 'http://bronyrock.com/track/the-pony-mash',
+            'file': '3235767654.mp3',
+            'info_dict': {
+                'title': 'The Pony Mash',
+                'uploader': 'M_Pallante',
             },
-            u'skip': u'There is a limit of 200 free downloads / month for the test song',
+            'skip': 'There is a limit of 200 free downloads / month for the test song',
         },
         # embedded brightcove video
         # it also tests brightcove videos that need to set the 'Referer' in the
         # http requests
         {
-            u'add_ie': ['Brightcove'],
-            u'url': u'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
-            u'info_dict': {
-                u'id': u'2765128793001',
-                u'ext': u'mp4',
-                u'title': u'Le cours de bourse : l’analyse technique',
-                u'description': u'md5:7e9ad046e968cb2d1114004aba466fd9',
-                u'uploader': u'BFM BUSINESS',
+            'add_ie': ['Brightcove'],
+            'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
+            'info_dict': {
+                'id': '2765128793001',
+                'ext': 'mp4',
+                'title': 'Le cours de bourse : l’analyse technique',
+                'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
+                'uploader': 'BFM BUSINESS',
             },
-            u'params': {
-                u'skip_download': True,
+            'params': {
+                'skip_download': True,
             },
         },
         # Direct link to a video
         {
-            u'url': u'http://media.w3.org/2010/05/sintel/trailer.mp4',
-            u'file': u'trailer.mp4',
-            u'md5': u'67d406c2bcb6af27fa886f31aa934bbe',
-            u'info_dict': {
-                u'id': u'trailer',
-                u'title': u'trailer',
-                u'upload_date': u'20100513',
+            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
+            'file': 'trailer.mp4',
+            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
+            'info_dict': {
+                'id': 'trailer',
+                'title': 'trailer',
+                'upload_date': '20100513',
             }
         },
         # ooyala video
         {
-            u'url': u'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
-            u'md5': u'5644c6ca5d5782c1d0d350dad9bd840c',
-            u'info_dict': {
-                u'id': u'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
-                u'ext': u'mp4',
-                u'title': u'2cc213299525360.mov', #that's what we get
+            'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
+            'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
+            'info_dict': {
+                'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
+                'ext': 'mp4',
+                'title': '2cc213299525360.mov', #that's what we get
             },
         },
     ]
@@ -101,12 +104,12 @@ class GenericIE(InfoExtractor):
     def report_download_webpage(self, video_id):
         """Report webpage download."""
         if not self._downloader.params.get('test', False):
-            self._downloader.report_warning(u'Falling back on generic information extractor.')
+            self._downloader.report_warning('Falling back on generic information extractor.')
         super(GenericIE, self).report_download_webpage(video_id)
 
     def report_following_redirect(self, new_url):
         """Report information extraction."""
-        self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
+        self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
 
     def _send_head(self, url):
         """Check if it is a redirect, like url shorteners, in case return the new url."""
@@ -152,7 +155,7 @@ class GenericIE(InfoExtractor):
 
         response = opener.open(HEADRequest(url))
         if response is None:
-            raise ExtractorError(u'Invalid URL protocol')
+            raise ExtractorError('Invalid URL protocol')
         return response
 
     def _real_extract(self, url):
@@ -162,7 +165,7 @@ class GenericIE(InfoExtractor):
             return self.url_result('http://' + url)
         video_id = os.path.splitext(url.split('/')[-1])[0]
 
-        self.to_screen(u'%s: Requesting header' % video_id)
+        self.to_screen('%s: Requesting header' % video_id)
 
         try:
             response = self._send_head(url)
@@ -186,7 +189,7 @@ class GenericIE(InfoExtractor):
                     'formats': [{
                         'format_id': m.group('format_id'),
                         'url': url,
-                        'vcodec': u'none' if m.group('type') == 'audio' else None
+                        'vcodec': 'none' if m.group('type') == 'audio' else None
                     }],
                     'upload_date': upload_date,
                 }
@@ -200,7 +203,7 @@ class GenericIE(InfoExtractor):
         except ValueError:
             # since this is the last-resort InfoExtractor, if
             # this error is thrown, it'll be thrown here
-            raise ExtractorError(u'Failed to download URL: %s' % url)
+            raise ExtractorError('Failed to download URL: %s' % url)
 
         self.report_extraction(video_id)
 
@@ -211,22 +214,23 @@ class GenericIE(InfoExtractor):
         #   Video Title - Tagline | Site Name
         # and so on and so forth; it's just not practical
         video_title = self._html_search_regex(
-            r'(?s)<title>(.*?)</title>', webpage, u'video title',
-            default=u'video')
+            r'(?s)<title>(.*?)</title>', webpage, 'video title',
+            default='video')
 
         # video uploader is domain name
         video_uploader = self._search_regex(
-            r'^(?:https?://)?([^/]*)/.*', url, u'video uploader')
+            r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
 
         # Look for BrightCove:
         bc_url = BrightcoveIE._extract_brightcove_url(webpage)
         if bc_url is not None:
-            self.to_screen(u'Brightcove video detected.')
-            return self.url_result(bc_url, 'Brightcove')
+            self.to_screen('Brightcove video detected.')
+            surl = smuggle_url(bc_url, {'Referer': url})
+            return self.url_result(surl, 'Brightcove')
 
         # Look for embedded (iframe) Vimeo player
         mobj = re.search(
-            r'<iframe[^>]+?src="(https?://player.vimeo.com/video/.+?)"', webpage)
+            r'<iframe[^>]+?src="((?:https?:)?//player.vimeo.com/video/.+?)"', webpage)
         if mobj:
             player_url = unescapeHTML(mobj.group(1))
             surl = smuggle_url(player_url, {'Referer': url})
@@ -299,10 +303,21 @@ class GenericIE(InfoExtractor):
             return OoyalaIE._build_url_result(mobj.group(1))
 
         # Look for Aparat videos
-        mobj = re.search(r'<iframe src="(http://www.aparat.com/video/[^"]+)"', webpage)
+        mobj = re.search(r'<iframe src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
         if mobj is not None:
             return self.url_result(mobj.group(1), 'Aparat')
 
+        # Look for MPORA videos
+        mobj = re.search(r'<iframe .*?src="(http://mpora\.com/videos/[^"]+)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group(1), 'Mpora')
+
+        # Look for embedded Novamov player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>http://(?:(?:embed|www)\.)?novamov\.com/embed\.php.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Novamov')
+
         # Start with something easy: JW Player in SWFObject
         mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
         if mobj is None:
@@ -313,7 +328,7 @@ class GenericIE(InfoExtractor):
             mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
         if mobj is None:
             # Broaden the search a little bit: JWPlayer JS loader
-            mobj = re.search(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http[^\'"]*)', webpage)
+            mobj = re.search(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage)
         if mobj is None:
             # Try to find twitter cards info
             mobj = re.search(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
@@ -328,23 +343,27 @@ class GenericIE(InfoExtractor):
             # HTML5 video
             mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)
         if mobj is None:
-            raise ExtractorError(u'Unsupported URL: %s' % url)
+            raise ExtractorError('Unsupported URL: %s' % url)
 
         # It's possible that one of the regexes
         # matched, but returned an empty group:
         if mobj.group(1) is None:
-            raise ExtractorError(u'Did not find a valid video URL at %s' % url)
+            raise ExtractorError('Did not find a valid video URL at %s' % url)
 
         video_url = mobj.group(1)
         video_url = compat_urlparse.urljoin(url, video_url)
         video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
 
+        # Sometimes, jwplayer extraction will result in a YouTube URL
+        if YoutubeIE.suitable(video_url):
+            return self.url_result(video_url, 'Youtube')
+
         # here's a fun little line of code for you:
         video_id = os.path.splitext(video_id)[0]
 
         return {
-            'id':       video_id,
-            'url':      video_url,
+            'id': video_id,
+            'url': video_url,
             'uploader': video_uploader,
-            'title':    video_title,
+            'title': video_title,
         }

youtube_dl/extractor/imdb.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json
 
@@ -9,18 +11,18 @@ from ..utils import (
 
 
 class ImdbIE(InfoExtractor):
-    IE_NAME = u'imdb'
-    IE_DESC = u'Internet Movie Database trailers'
+    IE_NAME = 'imdb'
+    IE_DESC = 'Internet Movie Database trailers'
     _VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'
 
     _TEST = {
-        u'url': u'http://www.imdb.com/video/imdb/vi2524815897',
-        u'md5': u'9f34fa777ade3a6e57a054fdbcb3a068',
-        u'info_dict': {
-            u'id': u'2524815897',
-            u'ext': u'mp4',
-            u'title': u'Ice Age: Continental Drift Trailer (No. 2) - IMDb',
-            u'description': u'md5:9061c2219254e5d14e03c25c98e96a81',
+        'url': 'http://www.imdb.com/video/imdb/vi2524815897',
+        'md5': '9f34fa777ade3a6e57a054fdbcb3a068',
+        'info_dict': {
+            'id': '2524815897',
+            'ext': 'mp4',
+            'title': 'Ice Age: Continental Drift Trailer (No. 2) - IMDb',
+            'description': 'md5:9061c2219254e5d14e03c25c98e96a81',
         }
     }
 
@@ -37,10 +39,10 @@ class ImdbIE(InfoExtractor):
             f_path = f_path.strip()
             format_page = self._download_webpage(
                 compat_urlparse.urljoin(url, f_path),
-                u'Downloading info for %s format' % f_id)
+                'Downloading info for %s format' % f_id)
             json_data = self._search_regex(
                 r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>',
-                format_page, u'json data', flags=re.DOTALL)
+                format_page, 'json data', flags=re.DOTALL)
             info = json.loads(json_data)
             format_info = info['videoPlayerObject']['video']
             formats.append({
@@ -56,9 +58,10 @@ class ImdbIE(InfoExtractor):
             'thumbnail': format_info['slate'],
         }
 
+
 class ImdbListIE(InfoExtractor):
-    IE_NAME = u'imdb:list'
-    IE_DESC = u'Internet Movie Database lists'
+    IE_NAME = 'imdb:list'
+    IE_DESC = 'Internet Movie Database lists'
     _VALID_URL = r'http://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
     
     def _real_extract(self, url):
@@ -66,13 +69,13 @@ class ImdbListIE(InfoExtractor):
         list_id = mobj.group('id')
         
         # RSS XML is sometimes malformed
-        rss = self._download_webpage('http://rss.imdb.com/list/%s' % list_id, list_id, u'Downloading list RSS')
-        list_title = self._html_search_regex(r'<title>(.*?)</title>', rss, u'list title')
+        rss = self._download_webpage('http://rss.imdb.com/list/%s' % list_id, list_id, 'Downloading list RSS')
+        list_title = self._html_search_regex(r'<title>(.*?)</title>', rss, 'list title')
         
         # Export is independent of actual author_id, but returns 404 if no author_id is provided.
         # However, passing dummy author_id seems to be enough.
         csv = self._download_webpage('http://www.imdb.com/list/export?list_id=%s&author_id=ur00000000' % list_id,
-                                     list_id, u'Downloading list CSV')
+                                     list_id, 'Downloading list CSV')
         
         entries = []
         for item in csv.split('\n')[1:]:
@@ -83,4 +86,4 @@ class ImdbListIE(InfoExtractor):
             if item_id.startswith('vi'):
                 entries.append(self.url_result('http://www.imdb.com/video/imdb/%s' % item_id, 'Imdb'))
         
-        return self.playlist_result(entries, list_id, list_title)
+        return self.playlist_result(entries, list_id, list_title)

youtube_dl/extractor/ina.py

@@ -7,7 +7,7 @@ class InaIE(InfoExtractor):
     """Information Extractor for Ina.fr"""
     _VALID_URL = r'(?:http://)?(?:www\.)?ina\.fr/video/(?P<id>I?[A-F0-9]+)/.*'
     _TEST = {
-        u'url': u'www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
+        u'url': u'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
         u'file': u'I12055569.mp4',
         u'md5': u'a667021bf2b41f8dc6049479d9bb38a3',
         u'info_dict': {

youtube_dl/extractor/khanacademy.py

@@ -0,0 +1,71 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+
+class KhanAcademyIE(InfoExtractor):
+    _VALID_URL = r'^https?://(?:www\.)?khanacademy\.org/(?P<key>[^/]+)/(?:[^/]+/){,2}(?P<id>[^?#/]+)(?:$|[?#])'
+    IE_NAME = 'KhanAcademy'
+
+    _TEST = {
+        'url': 'http://www.khanacademy.org/video/one-time-pad',
+        'file': 'one-time-pad.mp4',
+        'md5': '7021db7f2d47d4fff89b13177cb1e8f4',
+        'info_dict': {
+            'title': 'The one-time pad',
+            'description': 'The perfect cipher',
+            'duration': 176,
+            'uploader': 'Brit Cruise',
+            'upload_date': '20120411',
+        }
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+
+        if m.group('key') == 'video':
+            data = self._download_json(
+                'http://api.khanacademy.org/api/v1/videos/' + video_id,
+                video_id, 'Downloading video info')
+
+            upload_date = unified_strdate(data['date_added'])
+            uploader = ', '.join(data['author_names'])
+            return {
+                '_type': 'url_transparent',
+                'url': data['url'],
+                'id': video_id,
+                'title': data['title'],
+                'thumbnail': data['image_url'],
+                'duration': data['duration'],
+                'description': data['description'],
+                'uploader': uploader,
+                'upload_date': upload_date,
+            }
+        else:
+            # topic
+            data = self._download_json(
+                'http://api.khanacademy.org/api/v1/topic/' + video_id,
+                video_id, 'Downloading topic info')
+
+            entries = [
+                {
+                    '_type': 'url',
+                    'url': c['url'],
+                    'id': c['id'],
+                    'title': c['title'],
+                }
+                for c in data['children'] if c['kind'] in ('Video', 'Topic')]
+
+            return {
+                '_type': 'playlist',
+                'id': video_id,
+                'title': data['title'],
+                'description': data['description'],
+                'entries': entries,
+            }

youtube_dl/extractor/lynda.py

@@ -3,25 +3,40 @@ from __future__ import unicode_literals
 import re
 import json
 
+from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    compat_urllib_parse,
+    compat_urllib_request,
+    ExtractorError
+)
 
 
-class LyndaIE(InfoExtractor):
+class LyndaIE(SubtitlesInfoExtractor):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
     _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
+    _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
+    _NETRC_MACHINE = 'lynda'
+
+    _SUCCESSFUL_LOGIN_REGEX = r'<a href="https://www.lynda.com/home/userAccount/ChangeContactInfo.aspx" data-qa="eyebrow_account_menu">My account'
+    _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
+
+    ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
 
     _TEST = {
         'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
         'file': '114408.mp4',
         'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
-        u"info_dict": {
+        'info_dict': {
             'title': 'Using the exercise files',
             'duration': 68
         }
     }
 
+    def _real_initialize(self):
+        self._login()
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group(1)
@@ -30,11 +45,11 @@ class LyndaIE(InfoExtractor):
                                       video_id, 'Downloading video JSON')
         video_json = json.loads(page)
 
-        if 'Status' in video_json and video_json['Status'] == 'NotFound':
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+        if 'Status' in video_json:
+            raise ExtractorError('lynda returned error: %s' % video_json['Message'], expected=True)
 
         if video_json['HasAccess'] is False:
-            raise ExtractorError('Video %s is only available for members' % video_id, expected=True)
+            raise ExtractorError('Video %s is only available for members. ' % video_id + self.ACCOUNT_CREDENTIALS_HINT, expected=True)
 
         video_id = video_json['ID']
         duration = video_json['DurationInSeconds']
@@ -45,18 +60,100 @@ class LyndaIE(InfoExtractor):
                     'width': fmt['Width'],
                     'height': fmt['Height'],
                     'filesize': fmt['FileSize'],
-                    'format_id': fmt['Resolution']
+                    'format_id': str(fmt['Resolution'])
                     } for fmt in video_json['Formats']]
 
         self._sort_formats(formats)
 
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, page)
+            return
+
+        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, page))
+
         return {
             'id': video_id,
             'title': title,
             'duration': duration,
+            'subtitles': subtitles,
             'formats': formats
         }
 
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'username': username,
+            'password': password,
+            'remember': 'false',
+            'stayPut': 'false'
+        }        
+        request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+        login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+
+        # Not (yet) logged in
+        m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
+        if m is not None:
+            response = m.group('json')
+            response_json = json.loads(response)            
+            state = response_json['state']
+
+            if state == 'notlogged':
+                raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
+
+            # This is when we get popup:
+            # > You're already logged in to lynda.com on two devices.
+            # > If you log in here, we'll log you out of another device.
+            # So, we need to confirm this.
+            if state == 'conflicted':
+                confirm_form = {
+                    'username': '',
+                    'password': '',
+                    'resolve': 'true',
+                    'remember': 'false',
+                    'stayPut': 'false',
+                }
+                request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
+                login_page = self._download_webpage(request, None, note='Confirming log in and log out from another device')
+
+        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+            raise ExtractorError('Unable to log in')
+
+    def _fix_subtitles(self, subtitles):
+        if subtitles is None:
+            return subtitles  # subtitles not requested
+
+        fixed_subtitles = {}
+        for k, v in subtitles.items():
+            subs = json.loads(v)
+            if len(subs) == 0:
+                continue
+            srt = ''
+            for pos in range(0, len(subs) - 1):
+                seq_current = subs[pos]
+                m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
+                if m_current is None:
+                    continue
+                seq_next = subs[pos + 1]
+                m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
+                if m_next is None:
+                    continue
+                appear_time = m_current.group('timecode')
+                disappear_time = m_next.group('timecode')
+                text = seq_current['Caption']
+                srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
+            if srt:
+                fixed_subtitles[k] = srt
+        return fixed_subtitles
+
+    def _get_available_subtitles(self, video_id, webpage):
+        url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
+        sub = self._download_webpage(url, None, note=False)
+        sub_json = json.loads(sub)
+        return {'en': url} if len(sub_json) > 0 else {}
+
 
 class LyndaCourseIE(InfoExtractor):
     IE_NAME = 'lynda:course'
@@ -70,7 +167,7 @@ class LyndaCourseIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         course_path = mobj.group('coursepath')
         course_id = mobj.group('courseid')
-
+        
         page = self._download_webpage('http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
                                       course_id, 'Downloading course JSON')
         course_json = json.loads(page)
@@ -80,16 +177,18 @@ class LyndaCourseIE(InfoExtractor):
 
         unaccessible_videos = 0
         videos = []
+        (username, _) = self._get_login_info()
 
         for chapter in course_json['Chapters']:
             for video in chapter['Videos']:
-                if video['HasAccess'] is not True:
+                if username is None and video['HasAccess'] is False:
                     unaccessible_videos += 1
                     continue
                 videos.append(video['ID'])
 
         if unaccessible_videos > 0:
-            self._downloader.report_warning('%s videos are only available for members and will not be downloaded' % unaccessible_videos)
+            self._downloader.report_warning('%s videos are only available for members and will not be downloaded. '
+                                            % unaccessible_videos + LyndaIE.ACCOUNT_CREDENTIALS_HINT)
 
         entries = [
             self.url_result('http://www.lynda.com/%s/%s-4.html' %
@@ -99,4 +198,4 @@ class LyndaCourseIE(InfoExtractor):
 
         course_title = course_json['Title']
 
-        return self.playlist_result(entries, course_id, course_title)
+        return self.playlist_result(entries, course_id, course_title)

youtube_dl/extractor/metacritic.py

@@ -1,5 +1,6 @@
+from __future__ import unicode_literals
+
 import re
-import operator
 
 from .common import InfoExtractor
 from ..utils import (
@@ -11,12 +12,12 @@ class MetacriticIE(InfoExtractor):
     _VALID_URL = r'https?://www\.metacritic\.com/.+?/trailers/(?P<id>\d+)'
 
     _TEST = {
-        u'url': u'http://www.metacritic.com/game/playstation-4/infamous-second-son/trailers/3698222',
-        u'file': u'3698222.mp4',
-        u'info_dict': {
-            u'title': u'inFamous: Second Son - inSide Sucker Punch: Smoke & Mirrors',
-            u'description': u'Take a peak behind-the-scenes to see how Sucker Punch brings smoke into the universe of inFAMOUS Second Son on the PS4.',
-            u'duration': 221,
+        'url': 'http://www.metacritic.com/game/playstation-4/infamous-second-son/trailers/3698222',
+        'file': '3698222.mp4',
+        'info_dict': {
+            'title': 'inFamous: Second Son - inSide Sucker Punch: Smoke & Mirrors',
+            'description': 'Take a peak behind-the-scenes to see how Sucker Punch brings smoke into the universe of inFAMOUS Second Son on the PS4.',
+            'duration': 221,
         },
     }
 
@@ -26,7 +27,7 @@ class MetacriticIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         # The xml is not well formatted, there are raw '&'
         info = self._download_xml('http://www.metacritic.com/video_data?video=' + video_id,
-            video_id, u'Downloading info xml', transform_source=fix_xml_all_ampersand)
+            video_id, 'Downloading info xml', transform_source=fix_xml_all_ampersand)
 
         clip = next(c for c in info.findall('playList/clip') if c.find('id').text == video_id)
         formats = []
@@ -37,12 +38,12 @@ class MetacriticIE(InfoExtractor):
                 'url': video_url,
                 'ext': 'mp4',
                 'format_id': rate_str,
-                'rate': int(rate_str),
+                'tbr': int(rate_str),
             })
-        formats.sort(key=operator.itemgetter('rate'))
+        self._sort_formats(formats)
 
         description = self._html_search_regex(r'<b>Description:</b>(.*?)</p>',
-            webpage, u'description', flags=re.DOTALL)
+            webpage, 'description', flags=re.DOTALL)
 
         return {
             'id': video_id,

youtube_dl/extractor/mpora.py

@@ -0,0 +1,66 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+)
+
+
+class MporaIE(InfoExtractor):
+    _VALID_URL = r'^https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
+    IE_NAME = 'MPORA'
+
+    _TEST = {
+        'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
+        'file': 'AAdo8okx4wiz.mp4',
+        'md5': 'a7a228473eedd3be741397cf452932eb',
+        'info_dict': {
+            'title': 'Katy Curd -  Winter in the Forest',
+            'duration': 416,
+            'uploader': 'petenewman',
+        },
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+        data_json = self._search_regex(
+            r"new FM\.Player\('[^']+',\s*(\{.*?)\);\n", webpage, 'json')
+
+        data = json.loads(data_json)
+
+        uploader = data['info_overlay']['name']
+        duration = data['video']['duration'] // 1000
+        thumbnail = data['video']['encodings']['sd']['poster']
+        title = data['info_overlay']['title']
+
+        formats = []
+        for encoding_id, edata in data['video']['encodings'].items():
+            for src in edata['sources']:
+                width_str = self._search_regex(
+                    r'_([0-9]+)\.[a-zA-Z0-9]+$', src['src'],
+                    False, default=None)
+                vcodec = src['type'].partition('/')[2]
+                
+                formats.append({
+                    'format_id': encoding_id + '-' + vcodec,
+                    'url': src['src'],
+                    'vcodec': vcodec,
+                    'width': int_or_none(width_str),
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'uploader': uploader,
+            'duration': duration,
+            'thumbnail': thumbnail,
+        }

youtube_dl/extractor/novamov.py

@@ -0,0 +1,62 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    compat_urlparse
+)
+
+
+class NovamovIE(InfoExtractor):
+    _VALID_URL = r'http://(?:(?:www\.)?novamov\.com/video/|(?:(?:embed|www)\.)novamov\.com/embed\.php\?v=)(?P<videoid>[a-z\d]{13})'
+
+    _TEST = {
+        'url': 'http://www.novamov.com/video/4rurhn9x446jj',
+        'file': '4rurhn9x446jj.flv',
+        'md5': '7205f346a52bbeba427603ba10d4b935',
+        'info_dict': {
+            'title': 'search engine optimization',
+            'description': 'search engine optimization is used to rank the web page in the google search engine'
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('videoid')
+
+        page = self._download_webpage('http://www.novamov.com/video/%s' % video_id,
+                                      video_id, 'Downloading video page')
+
+        if re.search(r'This file no longer exists on our servers!</h2>', page) is not None:
+            raise ExtractorError(u'Video %s does not exist' % video_id, expected=True)
+
+        filekey = self._search_regex(
+            r'flashvars\.filekey="(?P<filekey>[^"]+)";', page, 'filekey')
+
+        title = self._html_search_regex(
+            r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>([^<]+)</h3>',
+            page, 'title', fatal=False)
+
+        description = self._html_search_regex(
+            r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>[^<]+</h3><p>([^<]+)</p>',
+            page, 'description', fatal=False)
+
+        api_response = self._download_webpage(
+            'http://www.novamov.com/api/player.api.php?key=%s&file=%s' % (filekey, video_id),
+            video_id, 'Downloading video api response')
+
+        response = compat_urlparse.parse_qs(api_response)
+
+        if 'error_msg' in response:
+            raise ExtractorError('novamov returned error: %s' % response['error_msg'][0], expected=True)
+
+        video_url = response['url'][0]
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': description
+        }

youtube_dl/extractor/nowvideo.py

@@ -5,7 +5,7 @@ from ..utils import compat_urlparse
 
 
 class NowVideoIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www\.)?nowvideo\.ch/video/(?P<id>\w+)'
+    _VALID_URL = r'(?:https?://)?(?:www\.)?nowvideo\.(?:ch|sx)/video/(?P<id>\w+)'
     _TEST = {
         u'url': u'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
         u'file': u'0mw0yow7b6dxa.flv',

youtube_dl/extractor/orf.py

@@ -1,54 +1,98 @@
 # coding: utf-8
+from __future__ import unicode_literals
 
-import re
-import xml.etree.ElementTree
 import json
+import re
 
 from .common import InfoExtractor
 from ..utils import (
-    compat_urlparse,
-    ExtractorError,
-    find_xpath_attr,
+    HEADRequest,
+    unified_strdate,
 )
 
+
 class ORFIE(InfoExtractor):
-    _VALID_URL = r'https?://tvthek\.orf\.at/(programs/.+?/episodes|topics/.+?)/(?P<id>\d+)'
+    _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://tvthek.orf.at/program/matinee-Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7317210/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319746/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319747',
+        'file': '7319747.mp4',
+        'md5': 'bd803c5d8c32d3c64a0ea4b4eeddf375',
+        'info_dict': {
+            'title': 'Was Sie schon immer über Klassik wissen wollten',
+            'description': 'md5:0ddf0d5f0060bd53f744edaa5c2e04a4',
+            'duration': 3508,
+            'upload_date': '20140105',
+        },
+        'skip': 'Blocked outside of Austria',
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         playlist_id = mobj.group('id')
         webpage = self._download_webpage(url, playlist_id)
 
-        flash_xml = self._search_regex('ORF.flashXML = \'(.+?)\'', webpage, u'flash xml')
-        flash_xml = compat_urlparse.parse_qs('xml='+flash_xml)['xml'][0]
-        flash_config = xml.etree.ElementTree.fromstring(flash_xml.encode('utf-8'))
-        playlist_json = self._search_regex(r'playlist\': \'(\[.*?\])\'', webpage, u'playlist').replace(r'\"','"')
-        playlist = json.loads(playlist_json)
+        data_json = self._search_regex(
+            r'initializeAdworx\((.+?)\);\n', webpage, 'video info')
+        all_data = json.loads(data_json)
+        sdata = all_data[0]['values']['segments']
 
-        videos = []
-        ns = '{http://tempuri.org/XMLSchema.xsd}'
-        xpath = '%(ns)sPlaylist/%(ns)sItems/%(ns)sItem' % {'ns': ns}
-        webpage_description = self._og_search_description(webpage)
-        for (i, (item, info)) in enumerate(zip(flash_config.findall(xpath), playlist), 1):
-            # Get best quality url
-            rtmp_url = None
-            for q in ['Q6A', 'Q4A', 'Q1A']:
-                video_url = find_xpath_attr(item, '%sVideoUrl' % ns, 'quality', q)
-                if video_url is not None:
-                    rtmp_url = video_url.text
-                    break
-            if rtmp_url is None:
-                raise ExtractorError(u'Couldn\'t get video url: %s' % info['id'])
-            description = self._html_search_regex(
-                r'id="playlist_entry_%s".*?<p>(.*?)</p>' % i, webpage,
-                u'description', default=webpage_description, flags=re.DOTALL)
-            videos.append({
+        def quality_to_int(s):
+            m = re.search('([0-9]+)', s)
+            if m is None:
+                return -1
+            return int(m.group(1))
+
+        entries = []
+        for sd in sdata:
+            video_id = sd['id']
+            formats = [{
+                'preference': -10 if fd['delivery'] == 'hls' else None,
+                'format_id': '%s-%s-%s' % (
+                    fd['delivery'], fd['quality'], fd['quality_string']),
+                'url': fd['src'],
+                'protocol': fd['protocol'],
+                'quality': quality_to_int(fd['quality']),
+            } for fd in sd['playlist_item_array']['sources']]
+
+            # Check for geoblocking.
+            # There is a property is_geoprotection, but that's always false
+            geo_str = sd.get('geoprotection_string')
+            if geo_str:
+                try:
+                    http_url = next(
+                        f['url']
+                        for f in formats
+                        if re.match(r'^https?://.*\.mp4$', f['url']))
+                except StopIteration:
+                    pass
+                else:
+                    req = HEADRequest(http_url)
+                    self._request_webpage(
+                        req, video_id,
+                        note='Testing for geoblocking',
+                        errnote=((
+                            'This video seems to be blocked outside of %s. '
+                            'You may want to try the streaming-* formats.')
+                            % geo_str),
+                        fatal=False)
+
+            self._sort_formats(formats)
+
+            upload_date = unified_strdate(sd['created_date'])
+            entries.append({
                 '_type': 'video',
-                'id': info['id'],
-                'title': info['title'],
-                'url': rtmp_url,
-                'ext': 'flv',
-                'description': description,
-                })
+                'id': video_id,
+                'title': sd['header'],
+                'formats': formats,
+                'description': sd.get('description'),
+                'duration': int(sd['duration_in_seconds']),
+                'upload_date': upload_date,
+                'thumbnail': sd.get('image_full_url'),
+            })
 
-        return videos
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': playlist_id,
+        }

youtube_dl/extractor/pornhub.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import re
 
@@ -11,16 +13,17 @@ from ..aes import (
     aes_decrypt_text
 )
 
+
 class PornHubIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>pornhub\.com/view_video\.php\?viewkey=(?P<videoid>[0-9a-f]+))'
     _TEST = {
-        u'url': u'http://www.pornhub.com/view_video.php?viewkey=648719015',
-        u'file': u'648719015.mp4',
-        u'md5': u'882f488fa1f0026f023f33576004a2ed',
-        u'info_dict': {
-            u"uploader": u"BABES-COM", 
-            u"title": u"Seductive Indian beauty strips down and fingers her pink pussy",
-            u"age_limit": 18
+        'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
+        'file': '648719015.mp4',
+        'md5': '882f488fa1f0026f023f33576004a2ed',
+        'info_dict': {
+            "uploader": "BABES-COM",
+            "title": "Seductive Indian beauty strips down and fingers her pink pussy",
+            "age_limit": 18
         }
     }
 
@@ -33,15 +36,15 @@ class PornHubIE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
-        video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, u'title')
-        video_uploader = self._html_search_regex(r'<b>From: </b>(?:\s|<[^>]*>)*(.+?)<', webpage, u'uploader', fatal=False)
-        thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, u'thumbnail', fatal=False)
+        video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
+        video_uploader = self._html_search_regex(r'<b>From: </b>(?:\s|<[^>]*>)*(.+?)<', webpage, 'uploader', fatal=False)
+        thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
         if thumbnail:
             thumbnail = compat_urllib_parse.unquote(thumbnail)
 
         video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:
-            password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, u'password').replace('+', ' ')
+            password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password').replace('+', ' ')
             video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
 
         formats = []
@@ -50,13 +53,24 @@ class PornHubIE(InfoExtractor):
             extension = os.path.splitext(path)[1][1:]
             format = path.split('/')[5].split('_')[:2]
             format = "-".join(format)
+
+            m = re.match(r'^(?P<height>[0-9]+)P-(?P<tbr>[0-9]+)K$', format)
+            if m is None:
+                height = None
+                tbr = None
+            else:
+                height = int(m.group('height'))
+                tbr = int(m.group('tbr'))
+
             formats.append({
                 'url': video_url,
                 'ext': extension,
                 'format': format,
                 'format_id': format,
+                'tbr': tbr,
+                'height': height,
             })
-        formats.sort(key=lambda format: list(map(lambda s: s.zfill(6), format['format'].split('-'))))
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

youtube_dl/extractor/redtube.py

@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class RedTubeIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
+    _VALID_URL = r'http://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
     _TEST = {
         u'url': u'http://www.redtube.com/66418',
         u'file': u'66418.mp4',

youtube_dl/extractor/rtlnow.py

@@ -39,7 +39,7 @@ class RTLnowIE(InfoExtractor):
         u'skip': u'Only works from Germany',
     },
     {
-        u'url': u'www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
+        u'url': u'http://www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
         u'file': u'13883.flv',
         u'info_dict': {
             u'upload_date': u'20090627', 

youtube_dl/extractor/veehd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json
 
@@ -8,16 +10,17 @@ from ..utils import (
     clean_html,
 )
 
+
 class VeeHDIE(InfoExtractor):
     _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
 
     _TEST = {
-        u'url': u'http://veehd.com/video/4686958',
-        u'file': u'4686958.mp4',
-        u'info_dict': {
-            u'title': u'Time Lapse View from Space ( ISS)',
-            u'uploader_id': u'spotted',
-            u'description': u'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
+        'url': 'http://veehd.com/video/4686958',
+        'file': '4686958.mp4',
+        'info_dict': {
+            'title': 'Time Lapse View from Space ( ISS)',
+            'uploader_id': 'spotted',
+            'description': 'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
         },
     }
 
@@ -25,24 +28,30 @@ class VeeHDIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        # VeeHD seems to send garbage on the first request.
+        # See https://github.com/rg3/youtube-dl/issues/2102
+        self._download_webpage(url, video_id, 'Requesting webpage')
         webpage = self._download_webpage(url, video_id)
-        player_path = self._search_regex(r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
-            webpage, u'player path')
+        player_path = self._search_regex(
+            r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
+            webpage, 'player path')
         player_url = compat_urlparse.urljoin(url, player_path)
-        player_page = self._download_webpage(player_url, video_id,
-            u'Downloading player page')
-        config_json = self._search_regex(r'value=\'config=({.+?})\'',
-            player_page, u'config json')
+
+        self._download_webpage(player_url, video_id, 'Requesting player page')
+        player_page = self._download_webpage(
+            player_url, video_id, 'Downloading player page')
+        config_json = self._search_regex(
+            r'value=\'config=({.+?})\'', player_page, 'config json')
         config = json.loads(config_json)
 
         video_url = compat_urlparse.unquote(config['clip']['url'])
         title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
         uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>',
-            webpage, u'uploader')
+            webpage, 'uploader')
         thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"',
-            webpage, u'thumbnail')
+            webpage, 'thumbnail')
         description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul',
-            webpage, u'description', flags=re.DOTALL)
+            webpage, 'description', flags=re.DOTALL)
 
         return {
             '_type': 'video',

youtube_dl/extractor/vimeo.py

@@ -1,4 +1,6 @@
 # encoding: utf-8
+from __future__ import unicode_literals
+
 import json
 import re
 import itertools
@@ -22,7 +24,7 @@ class VimeoIE(InfoExtractor):
 
     # _VALID_URL matches Vimeo URLs
     _VALID_URL = r'''(?x)
-        (?P<proto>https?://)?
+        (?P<proto>(?:https?:)?//)?
         (?:(?:www|(?P<player>player))\.)?
         vimeo(?P<pro>pro)?\.com/
         (?:.*?/)?
@@ -31,54 +33,55 @@ class VimeoIE(InfoExtractor):
         (?P<id>[0-9]+)
         /?(?:[?&].*)?(?:[#].*)?$'''
     _NETRC_MACHINE = 'vimeo'
-    IE_NAME = u'vimeo'
+    IE_NAME = 'vimeo'
     _TESTS = [
         {
-            u'url': u'http://vimeo.com/56015672#at=0',
-            u'file': u'56015672.mp4',
-            u'md5': u'8879b6cc097e987f02484baf890129e5',
-            u'info_dict': {
-                u"upload_date": u"20121220", 
-                u"description": u"This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550", 
-                u"uploader_id": u"user7108434", 
-                u"uploader": u"Filippo Valsorda", 
-                u"title": u"youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
+            'url': 'http://vimeo.com/56015672#at=0',
+            'file': '56015672.mp4',
+            'md5': '8879b6cc097e987f02484baf890129e5',
+            'info_dict': {
+                "upload_date": "20121220", 
+                "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550", 
+                "uploader_id": "user7108434", 
+                "uploader": "Filippo Valsorda", 
+                "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
             },
         },
         {
-            u'url': u'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
-            u'file': u'68093876.mp4',
-            u'md5': u'3b5ca6aa22b60dfeeadf50b72e44ed82',
-            u'note': u'Vimeo Pro video (#1197)',
-            u'info_dict': {
-                u'uploader_id': u'openstreetmapus', 
-                u'uploader': u'OpenStreetMap US', 
-                u'title': u'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
+            'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
+            'file': '68093876.mp4',
+            'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
+            'note': 'Vimeo Pro video (#1197)',
+            'info_dict': {
+                'uploader_id': 'openstreetmapus',
+                'uploader': 'OpenStreetMap US',
+                'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
             },
         },
         {
-            u'url': u'http://player.vimeo.com/video/54469442',
-            u'file': u'54469442.mp4',
-            u'md5': u'619b811a4417aa4abe78dc653becf511',
-            u'note': u'Videos that embed the url in the player page',
-            u'info_dict': {
-                u'title': u'Kathy Sierra: Building the minimum Badass User, Business of Software',
-                u'uploader': u'The BLN & Business of Software',
+            'url': 'http://player.vimeo.com/video/54469442',
+            'file': '54469442.mp4',
+            'md5': '619b811a4417aa4abe78dc653becf511',
+            'note': 'Videos that embed the url in the player page',
+            'info_dict': {
+                'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software',
+                'uploader': 'The BLN & Business of Software',
+                'uploader_id': 'theblnbusinessofsoftware',
             },
         },
         {
-            u'url': u'http://vimeo.com/68375962',
-            u'file': u'68375962.mp4',
-            u'md5': u'aaf896bdb7ddd6476df50007a0ac0ae7',
-            u'note': u'Video protected with password',
-            u'info_dict': {
-                u'title': u'youtube-dl password protected test video',
-                u'upload_date': u'20130614',
-                u'uploader_id': u'user18948128',
-                u'uploader': u'Jaime Marquínez Ferrándiz',
+            'url': 'http://vimeo.com/68375962',
+            'file': '68375962.mp4',
+            'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
+            'note': 'Video protected with password',
+            'info_dict': {
+                'title': 'youtube-dl password protected test video',
+                'upload_date': '20130614',
+                'uploader_id': 'user18948128',
+                'uploader': 'Jaime Marquínez Ferrándiz',
             },
-            u'params': {
-                u'videopassword': u'youtube-dl',
+            'params': {
+                'videopassword': 'youtube-dl',
             },
         },
     ]
@@ -90,7 +93,7 @@ class VimeoIE(InfoExtractor):
         self.report_login()
         login_url = 'https://vimeo.com/log_in'
         webpage = self._download_webpage(login_url, None, False)
-        token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1)
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
         data = compat_urllib_parse.urlencode({'email': username,
                                               'password': password,
                                               'action': 'login',
@@ -100,13 +103,13 @@ class VimeoIE(InfoExtractor):
         login_request = compat_urllib_request.Request(login_url, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Cookie', 'xsrft=%s' % token)
-        self._download_webpage(login_request, None, False, u'Wrong login info')
+        self._download_webpage(login_request, None, False, 'Wrong login info')
 
     def _verify_video_password(self, url, video_id, webpage):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
-            raise ExtractorError(u'This video is protected by a password, use the --video-password option')
-        token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1)
+            raise ExtractorError('This video is protected by a password, use the --video-password option')
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
         data = compat_urllib_parse.urlencode({'password': password,
                                               'token': token})
         # I didn't manage to use the password with https
@@ -118,8 +121,21 @@ class VimeoIE(InfoExtractor):
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Cookie', 'xsrft=%s' % token)
         self._download_webpage(password_request, video_id,
-                               u'Verifying the password',
-                               u'Wrong password')
+                               'Verifying the password',
+                               'Wrong password')
+
+    def _verify_player_video_password(self, url, video_id):
+        password = self._downloader.params.get('videopassword', None)
+        if password is None:
+            raise ExtractorError('This video is protected by a password, use the --video-password option')
+        data = compat_urllib_parse.urlencode({'password': password})
+        pass_url = url + '/check-password'
+        password_request = compat_urllib_request.Request(pass_url, data)
+        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        return self._download_json(
+            password_request, video_id,
+            'Verifying the password',
+            'Wrong password')
 
     def _real_initialize(self):
         self._login()
@@ -133,9 +149,6 @@ class VimeoIE(InfoExtractor):
 
         # Extract ID from URL
         mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
-
         video_id = mobj.group('id')
         if mobj.group('pro') or mobj.group('player'):
             url = 'http://player.vimeo.com/video/' + video_id
@@ -155,7 +168,7 @@ class VimeoIE(InfoExtractor):
         try:
             try:
                 config_url = self._html_search_regex(
-                    r' data-config-url="(.+?)"', webpage, u'config URL')
+                    r' data-config-url="(.+?)"', webpage, 'config URL')
                 config_json = self._download_webpage(config_url, video_id)
                 config = json.loads(config_json)
             except RegexNotFoundError:
@@ -166,19 +179,22 @@ class VimeoIE(InfoExtractor):
                     config_re = r'%s=({.+?});' % re.escape(m_variable_name.group(1))
                 else:
                     config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
-                config = self._search_regex(config_re, webpage, u'info section',
+                config = self._search_regex(config_re, webpage, 'info section',
                     flags=re.DOTALL)
                 config = json.loads(config)
         except Exception as e:
             if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
-                raise ExtractorError(u'The author has restricted the access to this video, try with the "--referer" option')
+                raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
             if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
                 self._verify_video_password(url, video_id, webpage)
                 return self._real_extract(url)
             else:
-                raise ExtractorError(u'Unable to extract info section',
+                raise ExtractorError('Unable to extract info section',
                                      cause=e)
+        else:
+            if config.get('view') == 4:
+                config = self._verify_player_video_password(url, video_id)
 
         # Extract title
         video_title = config["video"]["title"]
@@ -212,9 +228,9 @@ class VimeoIE(InfoExtractor):
             video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
 
         try:
-            view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, u'view count'))
-            like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, u'like count'))
-            comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, u'comment count'))
+            view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))
+            like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, 'like count'))
+            comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, 'comment count'))
         except RegexNotFoundError:
             # This info is only available in vimeo.com/{id} urls
             view_count = None
@@ -255,7 +271,7 @@ class VimeoIE(InfoExtractor):
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         if len(formats) == 0:
-            raise ExtractorError(u'No known codec found')
+            raise ExtractorError('No known codec found')
 
         return {
             'id':       video_id,
@@ -274,7 +290,7 @@ class VimeoIE(InfoExtractor):
 
 
 class VimeoChannelIE(InfoExtractor):
-    IE_NAME = u'vimeo:channel'
+    IE_NAME = 'vimeo:channel'
     _VALID_URL = r'(?:https?://)?vimeo.\com/channels/(?P<id>[^/]+)'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
     _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
@@ -283,14 +299,14 @@ class VimeoChannelIE(InfoExtractor):
         return '%s/videos/page:%d/' % (base_url, pagenum)
 
     def _extract_list_title(self, webpage):
-        return self._html_search_regex(self._TITLE_RE, webpage, u'list title')
+        return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
     def _extract_videos(self, list_id, base_url):
         video_ids = []
         for pagenum in itertools.count(1):
             webpage = self._download_webpage(
                 self._page_url(base_url, pagenum) ,list_id,
-                u'Downloading page %s' % pagenum)
+                'Downloading page %s' % pagenum)
             video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
@@ -310,8 +326,8 @@ class VimeoChannelIE(InfoExtractor):
 
 
 class VimeoUserIE(VimeoChannelIE):
-    IE_NAME = u'vimeo:user'
-    _VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)'
+    IE_NAME = 'vimeo:user'
+    _VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)(?:/videos|[#?]|$)'
     _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
 
     @classmethod
@@ -327,7 +343,7 @@ class VimeoUserIE(VimeoChannelIE):
 
 
 class VimeoAlbumIE(VimeoChannelIE):
-    IE_NAME = u'vimeo:album'
+    IE_NAME = 'vimeo:album'
     _VALID_URL = r'(?:https?://)?vimeo.\com/album/(?P<id>\d+)'
     _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
 
@@ -336,12 +352,12 @@ class VimeoAlbumIE(VimeoChannelIE):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        album_id =  mobj.group('id')
+        album_id = mobj.group('id')
         return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
 
 
 class VimeoGroupsIE(VimeoAlbumIE):
-    IE_NAME = u'vimeo:group'
+    IE_NAME = 'vimeo:group'
     _VALID_URL = r'(?:https?://)?vimeo.\com/groups/(?P<name>[^/]+)'
 
     def _extract_list_title(self, webpage):
@@ -351,3 +367,24 @@ class VimeoGroupsIE(VimeoAlbumIE):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
         return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name)
+
+
+class VimeoReviewIE(InfoExtractor):
+    IE_NAME = 'vimeo:review'
+    IE_DESC = 'Review pages on vimeo'
+    _VALID_URL = r'(?:https?://)?vimeo.\com/[^/]+/review/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
+        'file': '75524534.mp4',
+        'md5': 'c507a72f780cacc12b2248bb4006d253',
+        'info_dict': {
+            'title': "DICK HARDWICK 'Comedian'",
+            'uploader': 'Richard Hardwick',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        player_url = 'https://player.vimeo.com/player/' + video_id
+        return self.url_result(player_url, 'Vimeo', video_id)

youtube_dl/extractor/youtube.py

@@ -213,6 +213,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 48, 'preference': -50},
         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 256, 'preference': -50},
+
+        # RTMP (unnamed)
+        '_rtmp': {'protocol': 'rtmp'},
     }
 
     IE_NAME = u'youtube'
@@ -998,7 +1001,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'lang': lang,
                 'v': video_id,
                 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
-                'name': l[0].encode('utf-8'),
+                'name': unescapeHTML(l[0]).encode('utf-8'),
             })
             url = u'http://www.youtube.com/api/timedtext?' + params
             sub_lang_list[lang] = url
@@ -1273,7 +1276,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
             self.report_rtmp_download()
-            video_url_list = [(None, video_info['conn'][0])]
+            video_url_list = [('_rtmp', video_info['conn'][0])]
         elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
@@ -1759,6 +1762,6 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             u'Did you forget to quote the URL? Remember that & is a meta '
             u'character in most shells, so you want to put the URL in quotes, '
             u'like  youtube-dl '
-            u'\'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\''
-            u' (or simply  youtube-dl BaW_jenozKc  ).',
+            u'"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
+            u' or simply  youtube-dl BaW_jenozKc  .',
             expected=True)

youtube_dl/postprocessor/__init__.py

@@ -0,0 +1,18 @@
+
+from .ffmpeg import (
+    FFmpegMergerPP,
+    FFmpegMetadataPP,
+    FFmpegVideoConvertor,
+    FFmpegExtractAudioPP,
+    FFmpegEmbedSubtitlePP,
+)
+from .xattrpp import XAttrMetadataPP
+
+__all__ = [
+    'FFmpegMergerPP',
+    'FFmpegMetadataPP',
+    'FFmpegVideoConvertor',
+    'FFmpegExtractAudioPP',
+    'FFmpegEmbedSubtitlePP',
+    'XAttrMetadataPP',
+]

youtube_dl/postprocessor/common.py

@@ -0,0 +1,49 @@
+from ..utils import PostProcessingError
+
+
+class PostProcessor(object):
+    """Post Processor class.
+
+    PostProcessor objects can be added to downloaders with their
+    add_post_processor() method. When the downloader has finished a
+    successful download, it will take its internal chain of PostProcessors
+    and start calling the run() method on each one of them, first with
+    an initial argument and then with the returned value of the previous
+    PostProcessor.
+
+    The chain will be stopped if one of them ever returns None or the end
+    of the chain is reached.
+
+    PostProcessor objects follow a "mutual registration" process similar
+    to InfoExtractor objects.
+    """
+
+    _downloader = None
+
+    def __init__(self, downloader=None):
+        self._downloader = downloader
+
+    def set_downloader(self, downloader):
+        """Sets the downloader for this PP."""
+        self._downloader = downloader
+
+    def run(self, information):
+        """Run the PostProcessor.
+
+        The "information" argument is a dictionary like the ones
+        composed by InfoExtractors. The only difference is that this
+        one has an extra field called "filepath" that points to the
+        downloaded file.
+
+        This method returns a tuple, the first element of which describes
+        whether the original file should be kept (i.e. not deleted - None for
+        no preference), and the second of which is the updated information.
+
+        In addition, this method may raise a PostProcessingError
+        exception if post processing fails.
+        """
+        return None, information  # by default, keep file and do nothing
+
+
+class AudioConversionError(PostProcessingError):
+    pass

youtube_dl/postprocessor/ffmpeg.py

@@ -4,65 +4,23 @@ import sys
 import time
 
 
-from .utils import (
+from .common import AudioConversionError, PostProcessor
+
+from ..utils import (
+    check_executable,
     compat_subprocess_get_DEVNULL,
     encodeFilename,
     PostProcessingError,
+    prepend_extension,
     shell_quote,
     subtitles_filename,
-    prepend_extension,
 )
 
 
-class PostProcessor(object):
-    """Post Processor class.
-
-    PostProcessor objects can be added to downloaders with their
-    add_post_processor() method. When the downloader has finished a
-    successful download, it will take its internal chain of PostProcessors
-    and start calling the run() method on each one of them, first with
-    an initial argument and then with the returned value of the previous
-    PostProcessor.
-
-    The chain will be stopped if one of them ever returns None or the end
-    of the chain is reached.
-
-    PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects.
-    """
-
-    _downloader = None
-
-    def __init__(self, downloader=None):
-        self._downloader = downloader
-
-    def set_downloader(self, downloader):
-        """Sets the downloader for this PP."""
-        self._downloader = downloader
-
-    def run(self, information):
-        """Run the PostProcessor.
-
-        The "information" argument is a dictionary like the ones
-        composed by InfoExtractors. The only difference is that this
-        one has an extra field called "filepath" that points to the
-        downloaded file.
-
-        This method returns a tuple, the first element of which describes
-        whether the original file should be kept (i.e. not deleted - None for
-        no preference), and the second of which is the updated information.
-
-        In addition, this method may raise a PostProcessingError
-        exception if post processing fails.
-        """
-        return None, information # by default, keep file and do nothing
 
 class FFmpegPostProcessorError(PostProcessingError):
     pass
 
-class AudioConversionError(PostProcessingError):
-    pass
-
 class FFmpegPostProcessor(PostProcessor):
     def __init__(self,downloader=None):
         PostProcessor.__init__(self, downloader)
@@ -70,23 +28,26 @@ class FFmpegPostProcessor(PostProcessor):
 
     @staticmethod
     def detect_executables():
-        def executable(exe):
-            try:
-                subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-            except OSError:
-                return False
-            return exe
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((program, executable(program)) for program in programs)
+        return dict((program, check_executable(program, ['-version'])) for program in programs)
+
+    def _get_executable(self):
+        if self._downloader.params.get('prefer_ffmpeg', False):
+            return self._exes['ffmpeg'] or self._exes['avconv']
+        else:
+            return self._exes['avconv'] or self._exes['ffmpeg']
+
+    def _uses_avconv(self):
+        return self._get_executable() == self._exes['avconv']
 
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
-        if not self._exes['ffmpeg'] and not self._exes['avconv']:
+        if not self._get_executable():
             raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
 
         files_cmd = []
         for path in input_paths:
             files_cmd.extend(['-i', encodeFilename(path, True)])
-        cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y'] + files_cmd
+        cmd = ([self._get_executable(), '-y'] + files_cmd
                + opts +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 
@@ -108,6 +69,7 @@ class FFmpegPostProcessor(PostProcessor):
             return u'./' + fn
         return fn
 
+
 class FFmpegExtractAudioPP(FFmpegPostProcessor):
     def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
         FFmpegPostProcessor.__init__(self, downloader)
@@ -140,8 +102,6 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         return None
 
     def run_ffmpeg(self, path, out_path, codec, more_opts):
-        if not self._exes['ffmpeg'] and not self._exes['avconv']:
-            raise AudioConversionError('ffmpeg or avconv not found. Please install one.')
         if codec is None:
             acodec_opts = []
         else:
@@ -159,13 +119,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         if filecodec is None:
             raise PostProcessingError(u'WARNING: unable to obtain file audio codec with ffprobe')
 
+        uses_avconv = self._uses_avconv()
         more_opts = []
         if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
             if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
                 # Lossless, but in another container
                 acodec = 'copy'
                 extension = 'm4a'
-                more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+                more_opts = ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
             elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
                 # Lossless if possible
                 acodec = 'copy'
@@ -181,9 +142,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 more_opts = []
                 if self._preferredquality is not None:
                     if int(self._preferredquality) < 10:
-                        more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+                        more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
                     else:
-                        more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+                        more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
         else:
             # We convert the audio (lossy)
             acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
@@ -192,13 +153,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._preferredquality is not None:
                 # The opus codec doesn't support the -aq option
                 if int(self._preferredquality) < 10 and extension != 'opus':
-                    more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+                    more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
                 else:
-                    more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+                    more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
             if self._preferredcodec == 'aac':
                 more_opts += ['-f', 'adts']
             if self._preferredcodec == 'm4a':
-                more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+                more_opts += ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
             if self._preferredcodec == 'vorbis':
                 extension = 'ogg'
             if self._preferredcodec == 'wav':
@@ -216,14 +177,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
                 self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
             else:
-                self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
+                self._downloader.to_screen(u'[' + self._get_executable() + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
         except:
             etype,e,tb = sys.exc_info()
             if isinstance(e, AudioConversionError):
                 msg = u'audio conversion failed: ' + e.msg
             else:
-                msg = u'error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg')
+                msg = u'error running ' + self._get_executable()
             raise PostProcessingError(msg)
 
         # Try to update the date time for extracted audio file.
@@ -236,6 +197,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         information['filepath'] = new_path
         return self._nopostoverwrites,information
 
+
 class FFmpegVideoConvertor(FFmpegPostProcessor):
     def __init__(self, downloader=None,preferedformat=None):
         super(FFmpegVideoConvertor, self).__init__(downloader)
@@ -519,3 +481,4 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         args = ['-c', 'copy']
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
         return True, info
+

youtube_dl/postprocessor/xattrpp.py

@@ -0,0 +1,108 @@
+import os
+import subprocess
+import sys
+
+from .common import PostProcessor
+from ..utils import (
+    check_executable,
+    hyphenate_date,
+)
+
+
+class XAttrMetadataPP(PostProcessor):
+
+    #
+    # More info about extended attributes for media:
+    #   http://freedesktop.org/wiki/CommonExtendedAttributes/
+    #   http://www.freedesktop.org/wiki/PhreedomDraft/
+    #   http://dublincore.org/documents/usageguide/elements.shtml
+    #
+    # TODO:
+    #  * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
+    #  * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
+    #
+
+    def run(self, info):
+        """ Set extended attributes on downloaded file (if xattr support is found). """
+
+        # This mess below finds the best xattr tool for the job and creates a
+        # "write_xattr" function.
+        try:
+            # try the pyxattr module...
+            import xattr
+
+            def write_xattr(path, key, value):
+                return xattr.setxattr(path, key, value)
+
+        except ImportError:
+            if os.name == 'nt':
+                # Write xattrs to NTFS Alternate Data Streams:
+                # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
+                def write_xattr(path, key, value):
+                    assert ':' not in key
+                    assert os.path.exists(path)
+
+                    ads_fn = path + ":" + key
+                    with open(ads_fn, "wb") as f:
+                        f.write(value)
+            else:
+                user_has_setfattr = check_executable("setfattr", ['--version'])
+                user_has_xattr = check_executable("xattr", ['-h'])
+
+                if user_has_setfattr or user_has_xattr:
+
+                    def write_xattr(path, key, value):
+                        if user_has_setfattr:
+                            cmd = ['setfattr', '-n', key, '-v', value, path]
+                        elif user_has_xattr:
+                            cmd = ['xattr', '-w', key, value, path]
+
+                        subprocess.check_output(cmd)
+
+                else:
+                    # On Unix, and can't find pyxattr, setfattr, or xattr.
+                    if sys.platform.startswith('linux'):
+                        self._downloader.report_error(
+                            "Couldn't find a tool to set the xattrs. "
+                            "Install either the python 'pyxattr' or 'xattr' "
+                            "modules, or the GNU 'attr' package "
+                            "(which contains the 'setfattr' tool).")
+                    else:
+                        self._downloader.report_error(
+                            "Couldn't find a tool to set the xattrs. "
+                            "Install either the python 'xattr' module, "
+                            "or the 'xattr' binary.")
+
+        # Write the metadata to the file's xattrs
+        self._downloader.to_screen('[metadata] Writing metadata to file\'s xattrs')
+
+        filename = info['filepath']
+
+        try:
+            xattr_mapping = {
+                'user.xdg.referrer.url': 'webpage_url',
+                # 'user.xdg.comment':            'description',
+                'user.dublincore.title': 'title',
+                'user.dublincore.date': 'upload_date',
+                'user.dublincore.description': 'description',
+                'user.dublincore.contributor': 'uploader',
+                'user.dublincore.format': 'format',
+            }
+
+            for xattrname, infoname in xattr_mapping.items():
+
+                value = info.get(infoname)
+
+                if value:
+                    if infoname == "upload_date":
+                        value = hyphenate_date(value)
+
+                    byte_value = value.encode('utf-8')
+                    write_xattr(filename, xattrname, byte_value)
+
+            return True, info
+
+        except (subprocess.CalledProcessError, OSError):
+            self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
+            return False, info
+

youtube_dl/utils.py

@@ -764,6 +764,7 @@ def unified_strdate(date_str):
         '%Y-%m-%d',
         '%d/%m/%Y',
         '%Y/%m/%d %H:%M:%S',
+        '%Y-%m-%d %H:%M:%S',
         '%d.%m.%Y %H:%M',
         '%Y-%m-%dT%H:%M:%SZ',
         '%Y-%m-%dT%H:%M:%S.%fZ',
@@ -817,6 +818,15 @@ def date_from_str(date_str):
         return today + delta
     return datetime.datetime.strptime(date_str, "%Y%m%d").date()
     
+def hyphenate_date(date_str):
+    """
+    Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
+    match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
+    if match is not None:
+        return '-'.join(match.groups())
+    else:
+        return date_str
+
 class DateRange(object):
     """Represents a time interval between two dates"""
     def __init__(self, start=None, end=None):
@@ -1026,9 +1036,9 @@ def smuggle_url(url, data):
     return url + u'#' + sdata
 
 
-def unsmuggle_url(smug_url):
+def unsmuggle_url(smug_url, default=None):
     if not '#__youtubedl_smuggle' in smug_url:
-        return smug_url, None
+        return smug_url, default
     url, _, sdata = smug_url.rpartition(u'#')
     jsond = compat_parse_qs(sdata)[u'__youtubedl_smuggle'][0]
     data = json.loads(jsond)
@@ -1141,3 +1151,13 @@ def parse_duration(s):
 def prepend_extension(filename, ext):
     name, real_ext = os.path.splitext(filename) 
     return u'{0}.{1}{2}'.format(name, ext, real_ext)
+
+
+def check_executable(exe, args=[]):
+    """ Checks if the given binary is installed somewhere in PATH, and returns its name.
+    args can be a list of arguments for a short output (like -version) """
+    try:
+        subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+    except OSError:
+        return False
+    return exe

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.01.05.5'
+__version__ = '2013.01.17'