release 2013.04.31

[ci skip]

.travis.yml

@@ -8,6 +8,7 @@ notifications:
   email:
     - filippo.valsorda@gmail.com
     - phihag@phihag.de
+    - jaime.marquinez.ferrandiz+travis@gmail.com
 #  irc:
 #    channels:
 #      - "irc.freenode.org#youtube-dl"

README.md

@@ -27,8 +27,11 @@ which means you can modify it, redistribute it or use it however you like.
                                from an initial value of SIZE.
     --dump-user-agent          display the current browser identification
     --user-agent UA            specify a custom user agent
+    --referer REF              specify a custom referer, use if the video access
+                               is restricted to one domain
     --list-extractors          List all supported extractors and the URLs they
                                would handle
+    --proxy URL                Use the specified HTTP/HTTPS proxy
 
 ## Video Selection:
     --playlist-start NUMBER    playlist video to start at (default is 1)
@@ -42,10 +45,13 @@ which means you can modify it, redistribute it or use it however you like.
                                (e.g. 50k or 44.6m)
     --max-filesize SIZE        Do not download any videos larger than SIZE (e.g.
                                50k or 44.6m)
+    --date DATE                download only videos uploaded in this date
+    --datebefore DATE          download only videos uploaded before this date
+    --dateafter DATE           download only videos uploaded after this date
 
 ## Filesystem Options:
-    -t, --title                use title in file name
-    --id                       use video ID in file name
+    -t, --title                use title in file name (default)
+    --id                       use only video ID in file name
     -l, --literal              [deprecated] alias of --title
     -A, --auto-number          number downloaded files starting from 00000
     -o, --output TEMPLATE      output filename template. Use %(title)s to get
@@ -56,10 +62,12 @@ which means you can modify it, redistribute it or use it however you like.
                                extension, %(upload_date)s for the upload date
                                (YYYYMMDD), %(extractor)s for the provider
                                (youtube, metacafe, etc), %(id)s for the video id
-                               and %% for a literal percent. Use - to output to
-                               stdout. Can also be used to download to a
-                               different directory, for example with -o '/my/dow
-                               nloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
+                               , %(playlist)s for the playlist the video is in,
+                               %(playlist_index)s for the position in the
+                               playlist and %% for a literal percent. Use - to
+                               output to stdout. Can also be used to download to
+                               a different directory, for example with -o '/my/d
+                               ownloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
     --autonumber-size NUMBER   Specifies the number of digits in %(autonumber)s
                                when it is present in output filename template or
                                --autonumber option is given
@@ -96,7 +104,8 @@ which means you can modify it, redistribute it or use it however you like.
                                verbose)
 
 ## Video Format Options:
-    -f, --format FORMAT        video format code
+    -f, --format FORMAT        video format code, specifiy the order of
+                               preference using slashes: "-f 22/17/18"
     --all-formats              download all available video formats
     --prefer-free-formats      prefer free video formats unless a specific one
                                is requested
@@ -150,6 +159,8 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
  - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
+ - `playlist`: The name or the id of the playlist that contains the video.
+ - `playlist_index`: The index of the video in the playlist, a five-digit number.
 
 The current default template is `%(id)s.%(ext)s`, but that will be switchted to `%(title)s-%(id)s.%(ext)s` (which can be requested with `-t` at the moment).
 
@@ -160,6 +171,19 @@ In some cases, you don't want special characters such as 中, spaces, or &, such
     $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
     youtube-dl_test_video_.mp4          # A simple file name
 
+# VIDEO SELECTION
+
+Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:
+
+ - Absolute dates: Dates in the format `YYYYMMDD`.
+ - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?`
+ 
+Examples:
+
+	$ youtube-dl --dateafter now-6months #will only download the videos uploaded in the last 6 months
+	$ youtube-dl --date 19700101 #will only download the videos uploaded in January 1, 1970
+	$ youtube-dl --dateafter 20000101 --datebefore 20100101 #will only download the videos uploaded between 2000 and 2010
+
 # FAQ
 
 ### Can you please put the -b option back?

test/test_all_urls.py

@@ -7,7 +7,7 @@ import unittest
 import os
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE
 
 class TestAllURLsMatching(unittest.TestCase):
     def test_youtube_playlist_matching(self):
@@ -24,6 +24,11 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertTrue(YoutubeIE.suitable(u'PLtS2H6bU1M'))
         self.assertFalse(YoutubeIE.suitable(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
 
+    def test_youtube_channel_matching(self):
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM'))
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec'))
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM/videos'))
+
     def test_youtube_extract(self):
         self.assertEqual(YoutubeIE()._extract_id('http://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
         self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')

test/test_download.py

@@ -67,7 +67,7 @@ class TestDownload(unittest.TestCase):
 def generator(test_case):
 
     def test_template(self):
-        ie = getattr(youtube_dl.InfoExtractors, test_case['name'] + 'IE')
+        ie = youtube_dl.InfoExtractors.get_info_extractor(test_case['name'])
         if not ie._WORKING:
             print('Skipping: IE marked as not _WORKING')
             return

test/test_utils.py

@@ -14,6 +14,8 @@ from youtube_dl.utils import timeconvert
 from youtube_dl.utils import sanitize_filename
 from youtube_dl.utils import unescapeHTML
 from youtube_dl.utils import orderedSet
+from youtube_dl.utils import DateRange
+from youtube_dl.utils import unified_strdate
 
 if sys.version_info < (3, 0):
     _compat_str = lambda b: b.decode('unicode-escape')
@@ -95,6 +97,20 @@ class TestUtil(unittest.TestCase):
 
     def test_unescape_html(self):
         self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
+        
+    def test_daterange(self):
+        _20century = DateRange("19000101","20000101")
+        self.assertFalse("17890714" in _20century)
+        _ac = DateRange("00010101")
+        self.assertTrue("19690721" in _ac)
+        _firstmilenium = DateRange(end="10000101")
+        self.assertTrue("07110427" in _firstmilenium)
+
+    def test_unified_dates(self):
+        self.assertEqual(unified_strdate('December 21, 2010'), '20101221')
+        self.assertEqual(unified_strdate('8/7/2009'), '20090708')
+        self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214')
+        self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
 
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_lists.py

@@ -8,8 +8,9 @@ import json
 import os
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.InfoExtractors import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE
+from youtube_dl.InfoExtractors import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE
 from youtube_dl.utils import *
+from youtube_dl.FileDownloader import FileDownloader
 
 PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
 with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
@@ -22,64 +23,87 @@ proxy_handler = compat_urllib_request.ProxyHandler()
 opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
 compat_urllib_request.install_opener(opener)
 
-class FakeDownloader(object):
+class FakeDownloader(FileDownloader):
     def __init__(self):
         self.result = []
         self.params = parameters
     def to_screen(self, s):
         print(s)
-    def trouble(self, s):
+    def trouble(self, s, tb=None):
         raise Exception(s)
-    def download(self, x):
-        self.result.append(x)
+    def extract_info(self, url):
+        self.result.append(url)
+        return url
 
 class TestYoutubeLists(unittest.TestCase):
+    def assertIsPlaylist(self,info):
+        """Make sure the info has '_type' set to 'playlist'"""
+        self.assertEqual(info['_type'], 'playlist')
+
     def test_youtube_playlist(self):
         dl = FakeDownloader()
         ie = YoutubePlaylistIE(dl)
-        ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
-        ytie_results = [YoutubeIE()._extract_id(r[0]) for r in dl.result]
+        result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')[0]
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['title'], 'ytdl test PL')
+        ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
         self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE'])
 
     def test_issue_673(self):
         dl = FakeDownloader()
         ie = YoutubePlaylistIE(dl)
-        ie.extract('PLBB231211A4F62143')
-        self.assertTrue(len(dl.result) > 40)
+        result = ie.extract('PLBB231211A4F62143')[0]
+        self.assertEqual(result['title'], 'Team Fortress 2')
+        self.assertTrue(len(result['entries']) > 40)
 
     def test_youtube_playlist_long(self):
         dl = FakeDownloader()
         ie = YoutubePlaylistIE(dl)
-        ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
-        self.assertTrue(len(dl.result) >= 799)
+        result = ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')[0]
+        self.assertIsPlaylist(result)
+        self.assertTrue(len(result['entries']) >= 799)
 
     def test_youtube_playlist_with_deleted(self):
         #651
         dl = FakeDownloader()
         ie = YoutubePlaylistIE(dl)
-        ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')
-        ytie_results = [YoutubeIE()._extract_id(r[0]) for r in dl.result]
+        result = ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')[0]
+        ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
         self.assertFalse('pElCt5oNDuI' in ytie_results)
         self.assertFalse('KdPEApIVdWM' in ytie_results)
+        
+    def test_youtube_playlist_empty(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx')[0]
+        self.assertIsPlaylist(result)
+        self.assertEqual(len(result['entries']), 0)
 
     def test_youtube_course(self):
         dl = FakeDownloader()
         ie = YoutubePlaylistIE(dl)
         # TODO find a > 100 (paginating?) videos course
-        ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        self.assertEqual(YoutubeIE()._extract_id(dl.result[0][0]), 'j9WZyLZCBzs')
-        self.assertEqual(len(dl.result), 25)
-        self.assertEqual(YoutubeIE()._extract_id(dl.result[-1][0]), 'rYefUsYuEp0')
+        result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')[0]
+        entries = result['entries']
+        self.assertEqual(YoutubeIE()._extract_id(entries[0]['url']), 'j9WZyLZCBzs')
+        self.assertEqual(len(entries), 25)
+        self.assertEqual(YoutubeIE()._extract_id(entries[-1]['url']), 'rYefUsYuEp0')
 
     def test_youtube_channel(self):
-        # I give up, please find a channel that does paginate and test this like test_youtube_playlist_long
-        pass # TODO
+        dl = FakeDownloader()
+        ie = YoutubeChannelIE(dl)
+        #test paginated channel
+        result = ie.extract('https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w')[0]
+        self.assertTrue(len(result['entries']) > 90)
+        #test autogenerated channel
+        result = ie.extract('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')[0]
+        self.assertTrue(len(result['entries']) >= 18)
 
     def test_youtube_user(self):
         dl = FakeDownloader()
         ie = YoutubeUserIE(dl)
-        ie.extract('https://www.youtube.com/user/TheLinuxFoundation')
-        self.assertTrue(len(dl.result) >= 320)
+        result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation')[0]
+        self.assertTrue(len(result['entries']) >= 320)
 
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_subtitles.py

@@ -30,7 +30,7 @@ class FakeDownloader(object):
         self.params = parameters
     def to_screen(self, s):
         print(s)
-    def trouble(self, s):
+    def trouble(self, s, tb=None):
         raise Exception(s)
     def download(self, x):
         self.result.append(x)
@@ -80,7 +80,7 @@ class TestYoutubeSubtitles(unittest.TestCase):
         IE = YoutubeIE(DL)
         info_dict = IE.extract('QRS8MkLhQmM')
         subtitles = info_dict[0]['subtitles']
-        self.assertEqual(len(subtitles), 12)
+        self.assertEqual(len(subtitles), 13)
     def test_youtube_subtitles_format(self):
         DL = FakeDownloader()
         DL.params['writesubtitles'] = True

test/tests.json

@@ -337,6 +337,15 @@
         "title": "11.04.2013 09:23 Uhr - Tagesschau in 100 Sekunden"
     },
     "skip": "Requires rtmpdump"
+  },
+  {
+    "name": "Tumblr",
+    "url": "http://birthdayproject2012.tumblr.com/post/17258355236/a-sample-video-from-leeann-if-you-need-an-idea",
+    "file": "17258355236.mp4",
+    "md5": "7c6a514d691b034ccf8567999e9e88a3",
+    "info_dict": {
+        "title": "A sample video from LeeAnn. (If you need an idea..."
+    }
   }
 
 ]

youtube_dl/FileDownloader.py

@@ -17,6 +17,7 @@ if os.name == 'nt':
     import ctypes
 
 from .utils import *
+from .InfoExtractors import get_info_extractor
 
 
 class FileDownloader(object):
@@ -88,6 +89,7 @@ class FileDownloader(object):
     keepvideo:         Keep the video file after post-processing
     min_filesize:      Skip files smaller than this size
     max_filesize:      Skip files larger than this size
+    daterange:         A DateRange object, download only if the upload_date is in the range.
     """
 
     params = None
@@ -120,7 +122,7 @@ class FileDownloader(object):
             exponent = 0
         else:
             exponent = int(math.log(bytes, 1024.0))
-        suffix = 'bkMGTPEZY'[exponent]
+        suffix = ['B','KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB'][exponent]
         converted = float(bytes) / float(1024 ** exponent)
         return '%.2f%s' % (converted, suffix)
 
@@ -253,7 +255,7 @@ class FileDownloader(object):
         Print the message to stderr, it will be prefixed with 'WARNING:'
         If stderr is a tty file the 'WARNING:' will be colored
         '''
-        if sys.stderr.isatty():
+        if sys.stderr.isatty() and os.name != 'nt':
             _msg_header=u'\033[0;33mWARNING:\033[0m'
         else:
             _msg_header=u'WARNING:'
@@ -265,7 +267,7 @@ class FileDownloader(object):
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        if sys.stderr.isatty():
+        if sys.stderr.isatty() and os.name != 'nt':
             _msg_header = u'\033[0;31mERROR:\033[0m'
         else:
             _msg_header = u'ERROR:'
@@ -343,12 +345,13 @@ class FileDownloader(object):
         """Report download progress."""
         if self.params.get('noprogress', False):
             return
+        clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
         if self.params.get('progress_with_newline', False):
             self.to_screen(u'[download] %s of %s at %s ETA %s' %
                 (percent_str, data_len_str, speed_str, eta_str))
         else:
-            self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
-                (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
+            self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' %
+                (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
         self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 
@@ -393,6 +396,8 @@ class FileDownloader(object):
                 autonumber_size = 5
             autonumber_templ = u'%0' + str(autonumber_size) + u'd'
             template_dict['autonumber'] = autonumber_templ % self._num_downloads
+            if template_dict['playlist_index'] is not None:
+                template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
 
             sanitize = lambda k,v: sanitize_filename(
                 u'NA' if v is None else compat_str(v),
@@ -403,10 +408,10 @@ class FileDownloader(object):
             filename = self.params['outtmpl'] % template_dict
             return filename
         except KeyError as err:
-            self.trouble(u'ERROR: Erroneous output template')
+            self.report_error(u'Erroneous output template')
             return None
         except ValueError as err:
-            self.trouble(u'ERROR: Insufficient system charset ' + repr(preferredencoding()))
+            self.report_error(u'Insufficient system charset ' + repr(preferredencoding()))
             return None
 
     def _match_entry(self, info_dict):
@@ -421,11 +426,132 @@ class FileDownloader(object):
         if rejecttitle:
             if re.search(rejecttitle, title, re.IGNORECASE):
                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
+        date = info_dict.get('upload_date', None)
+        if date is not None:
+            dateRange = self.params.get('daterange', DateRange())
+            if date not in dateRange:
+                return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
         return None
+        
+    def extract_info(self, url, download = True, ie_name = None):
+        '''
+        Returns a list with a dictionary for each video we find.
+        If 'download', also downloads the videos.
+         '''
+        suitable_found = False
+        
+        #We copy the original list
+        ies = list(self._ies)
+
+        if ie_name is not None:
+            #We put in the first place the given info extractor
+            first_ie = get_info_extractor(ie_name)()
+            first_ie.set_downloader(self)
+            ies.insert(0, first_ie)
+
+        for ie in ies:
+            # Go to next InfoExtractor if not suitable
+            if not ie.suitable(url):
+                continue
+
+            # Warn if the _WORKING attribute is False
+            if not ie.working():
+                self.report_warning(u'the program functionality for this site has been marked as broken, '
+                               u'and will probably not work. If you want to go on, use the -i option.')
+
+            # Suitable InfoExtractor found
+            suitable_found = True
+
+            # Extract information from URL and process it
+            try:
+                ie_results = ie.extract(url)
+                if ie_results is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
+                    break
+                results = []
+                for ie_result in ie_results:
+                    if not 'extractor' in ie_result:
+                        #The extractor has already been set somewhere else
+                        ie_result['extractor'] = ie.IE_NAME
+                    results.append(self.process_ie_result(ie_result, download))
+                return results
+            except ExtractorError as de: # An error we somewhat expected
+                self.report_error(compat_str(de), de.format_traceback())
+                break
+            except Exception as e:
+                if self.params.get('ignoreerrors', False):
+                    self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
+                    break
+                else:
+                    raise
+        if not suitable_found:
+                self.report_error(u'no suitable InfoExtractor: %s' % url)
+        
+    def process_ie_result(self, ie_result, download = True):
+        """
+        Take the result of the ie and return a list of videos.
+        For url elements it will search the suitable ie and get the videos
+        For playlist elements it will process each of the elements of the 'entries' key
+        
+        It will also download the videos if 'download'.
+        """
+        result_type = ie_result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system
+        if result_type == 'video':
+            if 'playlist' not in ie_result:
+                #It isn't part of a playlist
+                ie_result['playlist'] = None
+                ie_result['playlist_index'] = None
+            if download:
+                #Do the download:
+                self.process_info(ie_result)
+            return ie_result
+        elif result_type == 'url':
+            #We get the video pointed by the url
+            result = self.extract_info(ie_result['url'], download, ie_name = ie_result['ie_key'])[0]
+            return result
+        elif result_type == 'playlist':
+            #We process each entry in the playlist
+            playlist = ie_result.get('title', None) or ie_result.get('id', None)
+            self.to_screen(u'[download] Downloading playlist: %s'  % playlist)
+
+            playlist_results = []
+
+            n_all_entries = len(ie_result['entries'])
+            playliststart = self.params.get('playliststart', 1) - 1
+            playlistend = self.params.get('playlistend', -1)
+
+            if playlistend == -1:
+                entries = ie_result['entries'][playliststart:]
+            else:
+                entries = ie_result['entries'][playliststart:playlistend]
+
+            n_entries = len(entries)
+
+            self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
+                (ie_result['extractor'], playlist, n_all_entries, n_entries))
+
+            for i,entry in enumerate(entries,1):
+                self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
+                entry_result = self.process_ie_result(entry, False)
+                entry_result['playlist'] = playlist
+                entry_result['playlist_index'] = i + playliststart
+                #We must do the download here to correctly set the 'playlist' key
+                if download:
+                    self.process_info(entry_result)
+                playlist_results.append(entry_result)
+            result = ie_result.copy()
+            result['entries'] = playlist_results
+            return result
 
     def process_info(self, info_dict):
         """Process a single dictionary returned by an InfoExtractor."""
 
+        #We increment the download the download count here to match the previous behaviour.
+        self.increment_downloads()
+        
+        info_dict['fulltitle'] = info_dict['title']
+        if len(info_dict['title']) > 200:
+            info_dict['title'] = info_dict['title'][:197] + u'...'
+
         # Keep for backwards compatibility
         info_dict['stitle'] = info_dict['title']
 
@@ -517,7 +643,7 @@ class FileDownloader(object):
                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
                                 subfile.write(sub)
                     except (OSError, IOError):
-                        self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
+                        self.report_error(u'Cannot write subtitles file ' + descfn)
                         return
             if self.params.get('onlysubtitles', False):
                 return 
@@ -560,53 +686,14 @@ class FileDownloader(object):
             raise SameFileError(self.params['outtmpl'])
 
         for url in url_list:
-            suitable_found = False
-            for ie in self._ies:
-                # Go to next InfoExtractor if not suitable
-                if not ie.suitable(url):
-                    continue
-
-                # Warn if the _WORKING attribute is False
-                if not ie.working():
-                    self.report_warning(u'the program functionality for this site has been marked as broken, '
-                                        u'and will probably not work. If you want to go on, use the -i option.')
-
-                # Suitable InfoExtractor found
-                suitable_found = True
-
-                # Extract information from URL and process it
-                try:
-                    videos = ie.extract(url)
-                except ExtractorError as de: # An error we somewhat expected
-                    self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
-                    break
-                except MaxDownloadsReached:
-                    self.to_screen(u'[info] Maximum number of downloaded files reached.')
-                    raise
-                except Exception as e:
-                    if self.params.get('ignoreerrors', False):
-                        self.report_error(u'' + compat_str(e), tb=compat_str(traceback.format_exc()))
-                        break
-                    else:
-                        raise
-
-                if len(videos or []) > 1 and self.fixed_template():
-                    raise SameFileError(self.params['outtmpl'])
-
-                for video in videos or []:
-                    video['extractor'] = ie.IE_NAME
-                    try:
-                        self.increment_downloads()
-                        self.process_info(video)
-                    except UnavailableVideoError:
-                        self.to_stderr(u"\n")
-                        self.report_error(u'unable to download video')
-
-                # Suitable InfoExtractor had been found; go to next URL
-                break
-
-            if not suitable_found:
-                self.report_error(u'no suitable InfoExtractor: %s' % url)
+            try:
+                #It also downloads the videos
+                videos = self.extract_info(url)
+            except UnavailableVideoError:
+                self.report_error(u'unable to download video')
+            except MaxDownloadsReached:
+                self.to_screen(u'[info] Maximum number of downloaded files reached.')
+                raise
 
         return self._download_retcode
 

youtube_dl/__init__.py

@@ -47,7 +47,7 @@ from .FileDownloader import *
 from .InfoExtractors import gen_extractors
 from .PostProcessor import *
 
-def parseOpts():
+def parseOpts(overrideArguments=None):
     def _readOptions(filename_bytes):
         try:
             optionf = open(filename_bytes)
@@ -140,9 +140,13 @@ def parseOpts():
             help='display the current browser identification', default=False)
     general.add_option('--user-agent',
             dest='user_agent', help='specify a custom user agent', metavar='UA')
+    general.add_option('--referer',
+            dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
+            metavar='REF', default=None)
     general.add_option('--list-extractors',
             action='store_true', dest='list_extractors',
             help='List all supported extractors and the URLs they would handle', default=False)
+    general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
     general.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 
     selection.add_option('--playlist-start',
@@ -154,6 +158,9 @@ def parseOpts():
     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
+    selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
+    selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
+    selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 
 
     authentication.add_option('-u', '--username',
@@ -165,7 +172,8 @@ def parseOpts():
 
 
     video_format.add_option('-f', '--format',
-            action='store', dest='format', metavar='FORMAT', help='video format code')
+            action='store', dest='format', metavar='FORMAT',
+            help='video format code, specifiy the order of preference using slashes: "-f 22/17/18"')
     video_format.add_option('--all-formats',
             action='store_const', dest='format', help='download all available video formats', const='all')
     video_format.add_option('--prefer-free-formats',
@@ -229,16 +237,25 @@ def parseOpts():
             help='print downloaded pages to debug problems(very verbose)')
 
     filesystem.add_option('-t', '--title',
-            action='store_true', dest='usetitle', help='use title in file name', default=False)
+            action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
     filesystem.add_option('--id',
-            action='store_true', dest='useid', help='use video ID in file name', default=False)
+            action='store_true', dest='useid', help='use only video ID in file name', default=False)
     filesystem.add_option('-l', '--literal',
             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
     filesystem.add_option('-A', '--auto-number',
             action='store_true', dest='autonumber',
             help='number downloaded files starting from 00000', default=False)
     filesystem.add_option('-o', '--output',
-            dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
+            dest='outtmpl', metavar='TEMPLATE',
+            help=('output filename template. Use %(title)s to get the title, '
+                  '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
+                  '%(autonumber)s to get an automatically incremented number, '
+                  '%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), '
+                  '%(extractor)s for the provider (youtube, metacafe, etc), '
+                  '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
+                  '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
+                  'Use - to output to stdout. Can also be used to download to a different directory, '
+                  'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
     filesystem.add_option('--autonumber-size',
             dest='autonumber_size', metavar='NUMBER',
             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given')
@@ -291,26 +308,30 @@ def parseOpts():
     parser.add_option_group(authentication)
     parser.add_option_group(postproc)
 
-    xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
-    if xdg_config_home:
-        userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
+    if overrideArguments is not None:
+        opts, args = parser.parse_args(overrideArguments)
+        if opts.verbose:
+            print(u'[debug] Override config: ' + repr(overrideArguments))
     else:
-        userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
-    systemConf = _readOptions('/etc/youtube-dl.conf')
-    userConf = _readOptions(userConfFile)
-    commandLineConf = sys.argv[1:]
-    argv = systemConf + userConf + commandLineConf
-    opts, args = parser.parse_args(argv)
-
-    if opts.verbose:
-        print(u'[debug] System config: ' + repr(systemConf))
-        print(u'[debug] User config: ' + repr(userConf))
-        print(u'[debug] Command-line args: ' + repr(commandLineConf))
+        xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
+        if xdg_config_home:
+            userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
+        else:
+            userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+        systemConf = _readOptions('/etc/youtube-dl.conf')
+        userConf = _readOptions(userConfFile)
+        commandLineConf = sys.argv[1:] 
+        argv = systemConf + userConf + commandLineConf
+        opts, args = parser.parse_args(argv)
+        if opts.verbose:
+            print(u'[debug] System config: ' + repr(systemConf))
+            print(u'[debug] User config: ' + repr(userConf))
+            print(u'[debug] Command-line args: ' + repr(commandLineConf))
 
     return parser, opts, args
 
-def _real_main():
-    parser, opts, args = parseOpts()
+def _real_main(argv=None):
+    parser, opts, args = parseOpts(argv)
 
     # Open appropriate CookieJar
     if opts.cookiefile is None:
@@ -328,6 +349,10 @@ def _real_main():
     # Set user agent
     if opts.user_agent is not None:
         std_headers['User-Agent'] = opts.user_agent
+    
+    # Set referer
+    if opts.referer is not None:
+        std_headers['Referer'] = opts.referer
 
     # Dump user agent
     if opts.dump_user_agent:
@@ -352,7 +377,14 @@ def _real_main():
 
     # General configuration
     cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-    proxy_handler = compat_urllib_request.ProxyHandler()
+    if opts.proxy:
+        proxies = {'http': opts.proxy, 'https': opts.proxy}
+    else:
+        proxies = compat_urllib_request.getproxies()
+        # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
+        if 'http' in proxies and 'https' not in proxies:
+            proxies['https'] = proxies['http']
+    proxy_handler = compat_urllib_request.ProxyHandler(proxies)
     opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
     compat_urllib_request.install_opener(opener)
     socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
@@ -426,6 +458,10 @@ def _real_main():
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
             parser.error(u'invalid video recode format specified')
+    if opts.date is not None:
+        date = DateRange.day(opts.date)
+    else:
+        date = DateRange(opts.dateafter, opts.datebefore)
 
     if sys.version_info < (3,):
         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
@@ -438,7 +474,7 @@ def _real_main():
             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
             or (opts.useid and u'%(id)s.%(ext)s')
             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
-            or u'%(id)s.%(ext)s')
+            or u'%(title)s-%(id)s.%(ext)s')
 
     # File downloader
     fd = FileDownloader({
@@ -492,7 +528,8 @@ def _real_main():
         'test': opts.test,
         'keepvideo': opts.keepvideo,
         'min_filesize': opts.min_filesize,
-        'max_filesize': opts.max_filesize
+        'max_filesize': opts.max_filesize,
+        'daterange': date
         })
 
     if opts.verbose:
@@ -544,9 +581,9 @@ def _real_main():
 
     sys.exit(retcode)
 
-def main():
+def main(argv=None):
     try:
-        _real_main()
+        _real_main(argv)
     except DownloadError:
         sys.exit(1)
     except SameFileError:

youtube_dl/update.py

@@ -78,7 +78,7 @@ def update_self(to_screen, verbose, filename):
     to_screen(u'Updating to version ' + versions_info['latest'] + '...')
     version = versions_info['versions'][versions_info['latest']]
 
-    print_notes(versions_info['versions'])
+    print_notes(to_screen, versions_info['versions'])
 
     if not os.access(filename, os.W_OK):
         to_screen(u'ERROR: no write permissions on %s' % filename)
@@ -157,11 +157,15 @@ del "%s"
 
     to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
 
-def print_notes(versions, fromVersion=__version__):
+def get_notes(versions, fromVersion):
     notes = []
     for v,vdata in sorted(versions.items()):
         if v > fromVersion:
             notes.extend(vdata.get('notes', []))
+    return notes
+
+def print_notes(to_screen, versions, fromVersion=__version__):
+    notes = get_notes(versions, fromVersion)
     if notes:
         to_screen(u'PLEASE NOTE:')
         for note in notes:

youtube_dl/utils.py

@@ -12,6 +12,7 @@ import traceback
 import zlib
 import email.utils
 import json
+import datetime
 
 try:
     import urllib.request as compat_urllib_request
@@ -568,3 +569,70 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     https_request = http_request
     https_response = http_response
+
+def unified_strdate(date_str):
+    """Return a string with the date in the format YYYYMMDD"""
+    upload_date = None
+    #Replace commas
+    date_str = date_str.replace(',',' ')
+    # %z (UTC offset) is only supported in python>=3.2
+    date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
+    format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S']
+    for expression in format_expressions:
+        try:
+            upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
+        except:
+            pass
+    return upload_date
+
+def date_from_str(date_str):
+    """
+    Return a datetime object from a string in the format YYYYMMDD or
+    (now|today)[+-][0-9](day|week|month|year)(s)?"""
+    today = datetime.date.today()
+    if date_str == 'now'or date_str == 'today':
+        return today
+    match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
+    if match is not None:
+        sign = match.group('sign')
+        time = int(match.group('time'))
+        if sign == '-':
+            time = -time
+        unit = match.group('unit')
+        #A bad aproximation?
+        if unit == 'month':
+            unit = 'day'
+            time *= 30
+        elif unit == 'year':
+            unit = 'day'
+            time *= 365
+        unit += 's'
+        delta = datetime.timedelta(**{unit: time})
+        return today + delta
+    return datetime.datetime.strptime(date_str, "%Y%m%d").date()
+    
+class DateRange(object):
+    """Represents a time interval between two dates"""
+    def __init__(self, start=None, end=None):
+        """start and end must be strings in the format accepted by date"""
+        if start is not None:
+            self.start = date_from_str(start)
+        else:
+            self.start = datetime.datetime.min.date()
+        if end is not None:
+            self.end = date_from_str(end)
+        else:
+            self.end = datetime.datetime.max.date()
+        if self.start > self.end:
+            raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
+    @classmethod
+    def day(cls, day):
+        """Returns a range that only contains the given day"""
+        return cls(day,day)
+    def __contains__(self, date):
+        """Check if the date is in the range"""
+        if not isinstance(date, datetime.date):
+            date = date_from_str(date)
+        return self.start <= date <= self.end
+    def __str__(self):
+        return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.04.11'
+__version__ = '2013.04.31'