mzitu | A multi-process , multi-threaded mzitu download script

# -*- coding: utf-8 -*- from scrapy_redis.spiders import RedisSpider from scrapy.spider import Request


from scrapy_redis_slaver.items import MzituSlaverItem

class MzituSpider(RedisSpider):
    name = 'mzitu'
    redis_key = 'mzitu:start_urls'    # get start url from redis
    def __init__(self, *args, **kwargs):
        self.item = MzituSlaverItem()

def parse(self, response):

    max_page = response.xpath(
        "descendant::div[@class='main']/div[@class='content']/div[@class='pagenavi']/a[last()-1]/span/text()").extract_first(default="N/A")
    max_page = int(max_page)
    name = response.xpath("./*//div[@class='main']/div[1]/h2/text()").extract_first(default="N/A")
    self.item['name'] = name
    self.item['url'] = response.url
    item_id = response.url.split('/')[-1]
    self.item['item_id'] = item_id
    # name:      the pictures' title
    # url:       the pictures' the first url
    # item_id:   the pictures' id
    # max_page:  the pictures' max page

    for num in range(1, max_page+1):         # The cycle is turning pages.
        # page_url is page address for each picture.
        page_url = response.url + '/' + str(num)
        yield Request(page_url, callback=self.img_url,meta={"name":name,
                                                            "item_id":item_id,
                                                            "max_page":max_page
                                                            })
def img_url(self, response):
    # this function:   get a picture's url from response
    img_urls = response.xpath("descendant::div[@class='main-image']/descendant::img/@src").extract_first()
    # a img_url
    self.server.sadd('{}:{}:images'.format(response.meta['name'], response.meta['item_id']), img_urls)
    # add a img_url to list of redis

    len_redis_img_list = self.server.scard('{}:{}:images'.format(response.meta['name'], response.meta['item_id']))
    # get the length of the img_url_list from redis

    if len_redis_img_list == response.meta['max_page']:
        self.item['img_urls'] = self.server.smembers('{}:{}:images'.format(response.meta['name'], response.meta['item_id']))
        print("yield item",response.meta['item_id'])
        yield self.item
    # in my mind,when the len_redis_img_list is equal the max_page,the item will be yield one
    # but actually,the item was yield the max_page times(very very more)