followed by

) art['images'] = [] for captionp in maindiv.findAll( 'p', {'class':'imageCaption'} ): img = captionp.findPrevious( 'img' ) if not img: continue img_caption = captionp.renderContents(None) img_credit = u'' # dailymail burns credit onto bottomleft of image img_url = img['src'] art['images'].append( {'url': img_url, 'caption': img_caption, 'credit': img_credit } ) # now extract article text # cruft removal for cruft in maindiv.findAll( 'h1' ): # empty h1 is a BeautifulSoup artifact cruft.extract() for cruft in maindiv.findAll( 'img' ): cruft.extract() for cruft in maindiv.findAll( 'p', {'class':'imageCaption'} ): cruft.extract() for cruft in maindiv.findAll( 'p', {'class':'scrollText'} ): cruft.extract() for cruft in maindiv.findAll( 'span', {'class':re.compile('^clickTo.*$') } ): cruft.extract() for cruft in maindiv.findAll( 'div', {'class':'clear'} ): cruft.extract() for cruft in maindiv.findAll( 'div', {'class':re.compile('^related.*$') } ): cruft.extract() for cruft in maindiv.findAll( 'div', {'class':re.compile('^thinFloat') } ): cruft.extract() for cruft in maindiv.findAll( 'div', {'class':'columnist-archive' } ): cruft.extract() for cruft in maindiv.findAll( 'div', {'class':'floatRHS' } ): cruft.extract() for cruft in maindiv.findAll( 'a', {'class':re.compile('^lightbox') } ): cruft.extract() for cruft in maindiv.findAll( 'div', {'class':re.compile('ArtInlineReadLinks') } ): cruft.extract() contenttxt = maindiv.prettify(None) contenttxt = contenttxt.replace( u'', u'' ) contenttxt = contenttxt.replace( u'', u'' ) contenttxt = ukmedia.SanitiseHTML( contenttxt ) art['content'] = contenttxt if desctxt == u'': desctxt = ukmedia.FirstPara( contenttxt ) desctxt = u' '.join( desctxt.split() ) art['description'] = desctxt return art def ScrubFunc( context, entry ): """mungefunc for ScraperUtils.FindArticlesFromRSS()""" # most dailymail RSS feeds go through feedburner, but luckily the original url is still there... url = context[ 'srcurl' ] url = TidyURL(url) if url.find('feedburner') != -1: url = entry.feedburner_origlink context['srcurl'] = url context['permalink'] = url context['srcid'] = CalcSrcID( url ) return context tidypat = re.compile( "^(.*?[.]html)(?:[?].*)?$" ) def TidyURL( url ): return tidypat.sub( r'\1', url ) # old style URLs: # http://www.dailymail.co.uk/pages/live/articles/news/news.html?in_article_id=564447 # new style (from late may 2008): # http://www.dailymail.co.uk/news/article-564447/Tories-ready-govern-moments-notice-insists-bullish-Cameron.html # # notes: # - article id is same (hooray!) # - old urls are redirected to new ones # - text after article id ignored (redirected to canonical url) # Canonical url form appears to be: # http://www.dailymail.co.uk/news/article-564447/index.html idpats = [ re.compile( r"\bin_article_id=(\d+)" ), re.compile( r"/article-(\d+)/.*[.]html" ) ] def CalcSrcID( url ): """ Generate a unique srcid from a url """ o = urlparse.urlparse( url ) # blogs are handled by blogs.py if o[1] not in ( 'www.dailymail.co.uk', 'www.mailonsunday.co.uk' ): return None for pat in idpats: m = pat.search( url ) if m: return 'dailymail_' + m.group(1) return None def ContextFromURL( url ): """Set up for scraping a single article from a bare url""" url = TidyURL(url) context = { 'srcurl': url, 'permalink': url, 'srcid': CalcSrcID( url ), 'srcorgname': u'dailymail', 'lastseen': datetime.now(), } return context def FindArticles(): """Look for recent articles""" rssfeeds = FindRSSFeeds( rss_feed_page ) found = ScraperUtils.FindArticlesFromRSS( rssfeeds, u'dailymail', ScrubFunc ) return found if __name__ == "__main__": ScraperUtils.RunMain( FindArticles, ContextFromURL, Extract, maxerrors=50 )