Compare commits

..

15 Commits

Author SHA1 Message Date
Ricardo Garcia
27c3383e2d Set version number 2010-10-31 11:24:08 +01:00
Ricardo Garcia
dbccb6cd84 Fix code for metacafe.com (this fixes issue #8) 2010-10-31 11:24:08 +01:00
Ricardo Garcia
98164eb3b9 Fix some minor unicode-related problems 2010-10-31 11:24:08 +01:00
Ricardo Garcia
2851b2ca18 Update internal documentation to reflect the new behaviour 2010-10-31 11:24:08 +01:00
Ricardo Garcia
6f21f68629 Download videos after extracting information
This is achieved by letting the InfoExtractors instruct its downloader to
process the information dictionary just after extracting the information. As a
consequence, some code is simplified too.
2010-10-31 11:24:08 +01:00
Ricardo Garcia
147753eb33 Replase self._downloader.to_stderr() with self._downloader.trouble() 2010-10-31 11:24:08 +01:00
Ricardo Garcia
3aaf887e98 Put the downloader in full control of output messages 2010-10-31 11:24:08 +01:00
Ricardo Garcia
9bf386d74b Move the downloader return code to a class member
This makes it possible to initialize it with value zero and later let the
trouble() overwrite the value. It simplifies error treatment and paves the way
for the InfoExtracto objects to call process_info() themselves, which should
solve the issues with tor and some other problems.
2010-10-31 11:24:08 +01:00
Ricardo Garcia
2f4d18a9f7 Use getpreferredencoding() instead of getdefaultlocale()
This fixes issue #7 and is recommended after a bug report I made to the
Python team:

http://bugs.python.org/issue5815
2010-10-31 11:24:08 +01:00
Ricardo Garcia
b0eddb2eb4 Update User-Agent string 2010-10-31 11:24:08 +01:00
Ricardo Garcia
9cee6d9035 Minor adjustments to closely match what a web browser does 2010-10-31 11:24:08 +01:00
Ricardo Garcia
c8619e0163 Move the code to process an InfoExtractor result to its own method 2010-10-31 11:24:08 +01:00
dannycolligan
257453b92b Added cap if user requests ytsearch number over 1000 (with warning) 2010-10-31 11:24:08 +01:00
dannyc@omega
fd9288c315 Changed ytsearchall to retrieve max 1000 results 2010-10-31 11:24:07 +01:00
Ricardo Garcia
1db4ff6054 Restore internal version number indicator 2010-10-31 11:24:07 +01:00

View File

@@ -19,7 +19,7 @@ import urllib
import urllib2
std_headers = {
'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5',
'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
'Accept-Language': 'en-us,en;q=0.5',
@@ -65,9 +65,10 @@ class FileDownloader(object):
For this, file downloader objects have a method that allows
InfoExtractors to be registered in a given order. When it is passed
a URL, the file downloader handles it to the first InfoExtractor it
finds that reports being able to handle it. The InfoExtractor returns
all the information to the FileDownloader and the latter downloads the
file or does whatever it's instructed to do.
finds that reports being able to handle it. The InfoExtractor extracts
all the information about the video or videos the URL refers to, and
asks the FileDownloader to process the video information, possibly
downloading the video.
File downloaders accept a lot of parameters. In order not to saturate
the object constructor with arguments, it receives a dictionary of
@@ -95,11 +96,13 @@ class FileDownloader(object):
params = None
_ies = []
_pps = []
_download_retcode = None
def __init__(self, params):
"""Create a FileDownloader object with the given options."""
self._ies = []
self._pps = []
self._download_retcode = 0
self.params = params
@staticmethod
@@ -187,7 +190,7 @@ class FileDownloader(object):
def to_stdout(self, message, skip_eol=False):
"""Print message to stdout if not in quiet mode."""
if not self.params.get('quiet', False):
print u'%s%s' % (message, [u'\n', u''][skip_eol]),
print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(locale.getpreferredencoding()),
sys.stdout.flush()
def to_stderr(self, message):
@@ -203,15 +206,13 @@ class FileDownloader(object):
Depending on if the downloader has been configured to ignore
download errors or not, this method may throw an exception or
not when errors are found, after printing the message. If it
doesn't raise, it returns an error code suitable to be returned
later as a program exit code to indicate error.
not when errors are found, after printing the message.
"""
if message is not None:
self.to_stderr(message)
if not self.params.get('ignoreerrors', False):
raise DownloadError(message)
return 1
self._download_retcode = 1
def slow_down(self, start_time, byte_counter):
"""Sleep if the download speed is over the rate limit."""
@@ -239,77 +240,78 @@ class FileDownloader(object):
"""Report download finished."""
self.to_stdout(u'')
def process_info(self, info_dict):
"""Process a single dictionary returned by an InfoExtractor."""
# Forced printings
if self.params.get('forcetitle', False):
print info_dict['title'].encode(locale.getpreferredencoding())
if self.params.get('forceurl', False):
print info_dict['url'].encode(locale.getpreferredencoding())
# Do nothing else if in simulate mode
if self.params.get('simulate', False):
return
try:
filename = self.params['outtmpl'] % info_dict
self.report_destination(filename)
except (ValueError, KeyError), err:
self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
if self.params['nooverwrites'] and os.path.exists(filename):
self.to_stderr('WARNING: file exists: %s; skipping' % filename)
return
try:
self.pmkdir(filename)
except (OSError, IOError), err:
self.trouble('ERROR: unable to create directories: %s' % str(err))
return
try:
outstream = open(filename, 'wb')
except (OSError, IOError), err:
self.trouble('ERROR: unable to open for writing: %s' % str(err))
return
try:
self._do_download(outstream, info_dict['url'])
outstream.close()
except (OSError, IOError), err:
self.trouble('ERROR: unable to write video data: %s' % str(err))
return
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.trouble('ERROR: unable to download video data: %s' % str(err))
return
try:
self.post_process(filename, info_dict)
except (PostProcessingError), err:
self.trouble('ERROR: postprocessing: %s' % str(err))
return
return
def download(self, url_list):
"""Download a given list of URLs."""
retcode = 0
if len(url_list) > 1 and self.fixed_template():
raise SameFileError(self.params['outtmpl'])
for url in url_list:
suitable_found = False
for ie in self._ies:
# Go to next InfoExtractor if not suitable
if not ie.suitable(url):
continue
# Suitable InfoExtractor found
suitable_found = True
all_results = ie.extract(url)
results = [x for x in all_results if x is not None]
if len(results) != len(all_results):
retcode = self.trouble()
if len(results) > 1 and self.fixed_template():
raise SameFileError(self.params['outtmpl'])
for result in results:
# Forced printings
if self.params.get('forcetitle', False):
print result['title']
if self.params.get('forceurl', False):
print result['url']
# Do nothing else if in simulate mode
if self.params.get('simulate', False):
continue
try:
filename = self.params['outtmpl'] % result
self.report_destination(filename)
except (ValueError, KeyError), err:
retcode = self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
continue
if self.params['nooverwrites'] and os.path.exists(filename):
self.to_stderr('WARNING: file exists: %s; skipping' % filename)
continue
try:
self.pmkdir(filename)
except (OSError, IOError), err:
retcode = self.trouble('ERROR: unable to create directories: %s' % str(err))
continue
try:
outstream = open(filename, 'wb')
except (OSError, IOError), err:
retcode = self.trouble('ERROR: unable to open for writing: %s' % str(err))
continue
try:
self._do_download(outstream, result['url'])
outstream.close()
except (OSError, IOError), err:
retcode = self.trouble('ERROR: unable to write video data: %s' % str(err))
continue
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
retcode = self.trouble('ERROR: unable to download video data: %s' % str(err))
continue
try:
self.post_process(filename, result)
except (PostProcessingError), err:
retcode = self.trouble('ERROR: postprocessing: %s' % str(err))
continue
# Extract information from URL and process it
ie.extract(url)
# Suitable InfoExtractor had been found; go to next URL
break
if not suitable_found:
retcode = self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
return retcode
if not suitable_found:
self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
return self._download_retcode
def post_process(self, filename, ie_info):
"""Run the postprocessing chain on the given file."""
@@ -359,9 +361,10 @@ class InfoExtractor(object):
Information extractors are the classes that, given a URL, extract
information from the video (or videos) the URL refers to. This
information includes the real video URL, the video title and simplified
title, author and others. It is returned in a list of dictionaries when
calling its extract() method. It is a list because a URL can refer to
more than one video (think of playlists). The dictionaries must include
title, author and others. The information is stored in a dictionary
which is then passed to the FileDownloader. The FileDownloader
processes this information possibly downloading the video to the file
system, among other possible outcomes. The dictionaries must include
the following fields:
id: Video identifier.
@@ -405,15 +408,6 @@ class InfoExtractor(object):
"""Sets the downloader for this IE."""
self._downloader = downloader
def to_stdout(self, message):
"""Print message to stdout if downloader is not in quiet mode."""
if self._downloader is None or not self._downloader.params.get('quiet', False):
print message
def to_stderr(self, message):
"""Print message to stderr."""
print >>sys.stderr, message
def _real_initialize(self):
"""Real initialization process. Redefine in subclasses."""
pass
@@ -460,27 +454,27 @@ class YoutubeIE(InfoExtractor):
def report_lang(self):
"""Report attempt to set language."""
self.to_stdout(u'[youtube] Setting language')
self._downloader.to_stdout(u'[youtube] Setting language')
def report_login(self):
"""Report attempt to log in."""
self.to_stdout(u'[youtube] Logging in')
self._downloader.to_stdout(u'[youtube] Logging in')
def report_age_confirmation(self):
"""Report attempt to confirm age."""
self.to_stdout(u'[youtube] Confirming age')
self._downloader.to_stdout(u'[youtube] Confirming age')
def report_webpage_download(self, video_id):
"""Report attempt to download webpage."""
self.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
def report_information_extraction(self, video_id):
"""Report attempt to extract video information."""
self.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
def report_video_url(self, video_id, video_real_url):
"""Report extracted video URL."""
self.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
def _real_initialize(self):
if self._downloader is None:
@@ -503,7 +497,7 @@ class YoutubeIE(InfoExtractor):
else:
raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
except (IOError, netrc.NetrcParseError), err:
self.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
return
# Set language
@@ -512,7 +506,7 @@ class YoutubeIE(InfoExtractor):
self.report_lang()
urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'WARNING: unable to set language: %s' % str(err))
self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
return
# No authentication to be performed
@@ -532,10 +526,10 @@ class YoutubeIE(InfoExtractor):
self.report_login()
login_results = urllib2.urlopen(request).read()
if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
self.to_stderr(u'WARNING: unable to log in: bad username or password')
self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
return
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'WARNING: unable to log in: %s' % str(err))
self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
return
# Confirm age
@@ -548,15 +542,15 @@ class YoutubeIE(InfoExtractor):
self.report_age_confirmation()
age_results = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to confirm age: %s' % str(err))
self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
return
def _real_extract(self, url):
# Extract video id from URL
mobj = re.match(self._VALID_URL, url)
if mobj is None:
self.to_stderr(u'ERROR: invalid URL: %s' % url)
return [None]
self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
return
video_id = mobj.group(2)
# Downloader parameters
@@ -581,16 +575,16 @@ class YoutubeIE(InfoExtractor):
self.report_webpage_download(video_id)
video_webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to download video webpage: %s' % str(err))
return [None]
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
return
self.report_information_extraction(video_id)
# "t" param
mobj = re.search(r', "t": "([^"]+)"', video_webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract "t" parameter')
return [None]
video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s' % (video_id, mobj.group(1))
self._downloader.trouble(u'ERROR: unable to extract "t" parameter')
return
video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&el=detailpage&ps=' % (video_id, mobj.group(1))
if format_param is not None:
video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
self.report_video_url(video_id, video_real_url)
@@ -598,15 +592,15 @@ class YoutubeIE(InfoExtractor):
# uploader
mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract uploader nickname')
return [None]
self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
return
video_uploader = mobj.group(1)
# title
mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>', video_webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract video title')
return [None]
self._downloader.trouble(u'ERROR: unable to extract video title')
return
video_title = mobj.group(1).decode('utf-8')
video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
video_title = video_title.replace(os.sep, u'%')
@@ -615,21 +609,22 @@ class YoutubeIE(InfoExtractor):
simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
simple_title = simple_title.strip(ur'_')
# Return information
return [{
# Process video information
self._downloader.process_info({
'id': video_id.decode('utf-8'),
'url': video_real_url.decode('utf-8'),
'uploader': video_uploader.decode('utf-8'),
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
}]
})
class MetacafeIE(InfoExtractor):
"""Information Extractor for metacafe.com."""
_VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
_DISCLAIMER = 'http://www.metacafe.com/family_filter/'
_FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
_youtube_ie = None
def __init__(self, youtube_ie, downloader=None):
@@ -642,19 +637,19 @@ class MetacafeIE(InfoExtractor):
def report_disclaimer(self):
"""Report disclaimer retrieval."""
self.to_stdout(u'[metacafe] Retrieving disclaimer')
self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
def report_age_confirmation(self):
"""Report attempt to confirm age."""
self.to_stdout(u'[metacafe] Confirming age')
self._downloader.to_stdout(u'[metacafe] Confirming age')
def report_download_webpage(self, video_id):
"""Report webpage download."""
self.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
def report_extraction(self, video_id):
"""Report information extraction."""
self.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
def _real_initialize(self):
# Retrieve disclaimer
@@ -663,7 +658,7 @@ class MetacafeIE(InfoExtractor):
self.report_disclaimer()
disclaimer = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
return
# Confirm age
@@ -671,27 +666,28 @@ class MetacafeIE(InfoExtractor):
'filters': '0',
'submit': "Continue - I'm over 18",
}
request = urllib2.Request('http://www.metacafe.com/', urllib.urlencode(disclaimer_form), std_headers)
request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
try:
self.report_age_confirmation()
disclaimer = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to confirm age: %s' % str(err))
self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
return
def _real_extract(self, url):
# Extract id and simplified title from URL
mobj = re.match(self._VALID_URL, url)
if mobj is None:
self.to_stderr(u'ERROR: invalid URL: %s' % url)
return [None]
self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
return
video_id = mobj.group(1)
# Check if video comes from YouTube
mobj2 = re.match(r'^yt-(.*)$', video_id)
if mobj2 is not None:
return self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
return
simple_title = mobj.group(2).decode('utf-8')
video_extension = 'flv'
@@ -702,46 +698,46 @@ class MetacafeIE(InfoExtractor):
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable retrieve video webpage: %s' % str(err))
return [None]
self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
return
# Extract URL, uploader and title from webpage
self.report_extraction(video_id)
mobj = re.search(r'(?m)"mediaURL":"(http.*?\.flv)"', webpage)
mobj = re.search(r'(?m)&mediaURL=(http.*?\.flv)', webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract media URL')
return [None]
mediaURL = mobj.group(1).replace('\\', '')
self._downloader.trouble(u'ERROR: unable to extract media URL')
return
mediaURL = urllib.unquote(mobj.group(1))
mobj = re.search(r'(?m)"gdaKey":"(.*?)"', webpage)
mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract gdaKey')
return [None]
self._downloader.trouble(u'ERROR: unable to extract gdaKey')
return
gdaKey = mobj.group(1)
video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract title')
return [None]
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
mobj = re.search(r'(?m)<li id="ChnlUsr">.*?Submitter:<br />(.*?)</li>', webpage)
mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract uploader nickname')
return [None]
video_uploader = re.sub(r'<.*?>', '', mobj.group(1))
self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
return
video_uploader = mobj.group(1)
# Return information
return [{
# Process video information
self._downloader.process_info({
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader.decode('utf-8'),
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
}]
})
class YoutubeSearchIE(InfoExtractor):
@@ -751,6 +747,7 @@ class YoutubeSearchIE(InfoExtractor):
_VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
_MORE_PAGES_INDICATOR = r'>Next</a>'
_youtube_ie = None
_max_youtube_results = 1000
def __init__(self, youtube_ie, downloader=None):
InfoExtractor.__init__(self, downloader)
@@ -762,7 +759,7 @@ class YoutubeSearchIE(InfoExtractor):
def report_download_page(self, query, pagenum):
"""Report attempt to download playlist page with given number."""
self.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
def _real_initialize(self):
self._youtube_ie.initialize()
@@ -770,24 +767,31 @@ class YoutubeSearchIE(InfoExtractor):
def _real_extract(self, query):
mobj = re.match(self._VALID_QUERY, query)
if mobj is None:
self.to_stderr(u'ERROR: invalid search query "%s"' % query)
return [None]
self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
return
prefix, query = query.split(':')
prefix = prefix[8:]
if prefix == '':
return self._download_n_results(query, 1)
self._download_n_results(query, 1)
return
elif prefix == 'all':
return self._download_n_results(query, -1)
self._download_n_results(query, self._max_youtube_results)
return
else:
try:
n = int(prefix)
if n <= 0:
self.to_stderr(u'ERROR: invalid download number %s for query "%s"' % (n, query))
return [None]
return self._download_n_results(query, n)
self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
return
elif n > self._max_youtube_results:
self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
n = self._max_youtube_results
self._download_n_results(query, n)
return
except ValueError: # parsing prefix as int fails
return self._download_n_results(query, 1)
self._download_n_results(query, 1)
return
def _download_n_results(self, query, n):
"""Downloads a specified number of results for a query"""
@@ -803,8 +807,8 @@ class YoutubeSearchIE(InfoExtractor):
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to download webpage: %s' % str(err))
return [None]
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
return
# Extract video identifiers
for mobj in re.finditer(self._VIDEO_INDICATOR, page):
@@ -814,16 +818,14 @@ class YoutubeSearchIE(InfoExtractor):
already_seen.add(video_id)
if len(video_ids) == n:
# Specified n videos reached
information = []
for id in video_ids:
information.extend(self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id))
return information
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
return
if self._MORE_PAGES_INDICATOR not in page:
information = []
for id in video_ids:
information.extend(self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id))
return information
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
return
pagenum = pagenum + 1
@@ -846,7 +848,7 @@ class YoutubePlaylistIE(InfoExtractor):
def report_download_page(self, playlist_id, pagenum):
"""Report attempt to download playlist page with given number."""
self.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
def _real_initialize(self):
self._youtube_ie.initialize()
@@ -855,8 +857,8 @@ class YoutubePlaylistIE(InfoExtractor):
# Extract playlist id
mobj = re.match(self._VALID_URL, url)
if mobj is None:
self.to_stderr(u'ERROR: invalid url: %s' % url)
return [None]
self._downloader.trouble(u'ERROR: invalid url: %s' % url)
return
# Download playlist pages
playlist_id = mobj.group(1)
@@ -869,8 +871,8 @@ class YoutubePlaylistIE(InfoExtractor):
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to download webpage: %s' % str(err))
return [None]
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
return
# Extract video identifiers
ids_in_page = []
@@ -883,10 +885,9 @@ class YoutubePlaylistIE(InfoExtractor):
break
pagenum = pagenum + 1
information = []
for id in video_ids:
information.extend(self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id))
return information
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
return
class PostProcessor(object):
"""Post Processor class.
@@ -910,15 +911,6 @@ class PostProcessor(object):
def __init__(self, downloader=None):
self._downloader = downloader
def to_stdout(self, message):
"""Print message to stdout if downloader is not in quiet mode."""
if self._downloader is None or not self._downloader.params.get('quiet', False):
print message
def to_stderr(self, message):
"""Print message to stderr."""
print >>sys.stderr, message
def set_downloader(self, downloader):
"""Sets the downloader for this PP."""
self._downloader = downloader
@@ -958,7 +950,7 @@ if __name__ == '__main__':
# Parse command line
parser = optparse.OptionParser(
usage='Usage: %prog [options] url...',
version='2009.04.06',
version='2009.04.25',
conflict_handler='resolve',
)
parser.add_option('-h', '--help',
@@ -1036,9 +1028,6 @@ if __name__ == '__main__':
youtube_search_ie = YoutubeSearchIE(youtube_ie)
# File downloader
charset = locale.getdefaultlocale()[1]
if charset is None:
charset = 'ascii'
fd = FileDownloader({
'usenetrc': opts.usenetrc,
'username': opts.username,
@@ -1048,7 +1037,7 @@ if __name__ == '__main__':
'forcetitle': opts.gettitle,
'simulate': (opts.simulate or opts.geturl or opts.gettitle),
'format': opts.format,
'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(charset))
'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
or u'%(id)s.%(ext)s'),