Question

I have this code. When i am using yyield to request futher link then i get this errror

Spider must return Request, BaseItem or None, got 'dict' 

I have tried verything but i can’t get rid of error

Code is here

def parse_items(self, response):
    hxs = HtmlXPathSelector(response)
    sites = hxs.select("//li[contains(concat(' ', @class, ' '), ' mod-searchresult-entry ')]")
    items = []


    for site in sites[:2]:

        item = SeekItem()
        item['title'] = myfilter(site.select('dl/dd/h2/a').select("string()").extract())
        item['link_url'] = myfilter(site.select('dl/dd/h2/em').select("string()").extract())
        item['description'] = myfilter(site.select('dl/dd/p').select("string()").extract())
        if  item['link_url']:
                      yield Request(urljoin('http://www.seek.com.au/', item['link_url']),
                      meta = item,
                      callback = self.parseItemDescription)

        yield item

def parseItemDescription(self, response):

    item = response.meta
    hxs = HtmlXPathSelector(response)
    sites = hxs.select("//li[contains(concat(' ', @class, ' '), ' mod-searchresult-entry ')]")
    item['description'] = "mytest"

    return item

Answer 1

Which version of scrapy are you using. The documentation for 0.16.2 has this method of passing items to another callback.

def parse_items(self, response):
    hxs = HtmlXPathSelector(response)
    sites = hxs.select("//li[contains(concat(' ', @class, ' '), ' mod-searchresult-entry ')]")
    items = []

    for site in sites[:2]:
        item = SeekItem()
        item['title'] = myfilter(site.select('dl/dd/h2/a').select("string()").extract())
        item['link_url'] = myfilter(site.select('dl/dd/h2/em').select("string()").extract())
        item['description'] = myfilter(site.select('dl/dd/p').select("string()").extract())
        if item['link_url']:
            request = Request("http://www.example.com/some_page.html", callback=self.parseItemDescription)
            request.meta['item'] = item
            return request

def parseItemDescription(self, response):

    item = response.meta['item']
    hxs = HtmlXPathSelector(response)
    sites = hxs.select("//li[contains(concat(' ', @class, ' '), ' mod-searchresult-entry ')]")
    item['description'] = "mytest"

    return item

NB: this is untested as the rest of your code (spider, items.py etc) is missing and I’m not sure how this is being run