Question

I’m trying to get the scrapy pipeline working to funnel data into my db.
The problem I’m having is being unable to store a value for the primary field in my table that is “contentid.’ I’m using the variable current_affiliate_item but not having any luck.

Thanks in advance for your help!

File "/Users/pipelines.py", line 46, in _conditional_insert
    self.update_affiliate_item(tx, item, affiliate_item)
  File "/Users/pipelines.py", line 109, in update_affiliate_item
    current_affiliate_item['id'], #used to be this - current_affiliate_item['id']
exceptions.KeyError: 'id'

This is the code it is calling – beginning with line 46
ln 46 is – self.update_affiliate_item(tx, item, affiliate_item)
ln 109 is – current_affiliate_item[‘id’]

I don’t know what my problem is here, and have been stumped for days on it. I’m trying to call

        if affiliate_item:
        self.update_affiliate_item(tx, item, affiliate_item)
        item_affiliate_id = affiliate_item['id']
        item['affiliate_item_id'] = affiliate_item['id']
        item['id'] = affiliate_item['item_id'] 
    else:
        item['id'] = self.get_item_id(tx, item)
        item_affiliate_id = self.insert_affiliate_item(tx, item)

    # item_affiliate_id = self.insert_or_update_affiliate(tx, item)
    self.insert_or_update_photos(tx, item)

    for price in item['prices']: 
        self.insert_or_update_price(tx, item, item_affiliate_id, price)

    # log.msg("Item prices stored in db: %s" % item['name'], level=log.INFO)

def find_affiliate_item(self, tx, item): #changed selx to self
    tx.execute("select * from z2ah5_jreviews_content where affiliate_item_id = %s and jr_retailer = %s", (item['affiliate_item_id'], item['retailer']))
    result = tx.fetchone()
    if result:
        return result
    else:
        return False

def get_item_id(self, tx, item):
    tx.execute("select * from z2ah5_jreviews_content where slug = %s ", (item['slug']))
    result = tx.fetchone()
    if result:
        return result['contentid']
    else:
        tx.execute(\
            "insert into z2ah5_jreviews_content (name, slug, date_created, date_modified) "
            "values (%s, %s, %s, %s)",
            (
                item['name'],
                item['slug'],
                datetime.datetime.now(),
                datetime.datetime.now(), 
                )
            )
        return tx.lastrowid

def insert_affiliate_item(self, tx, item):
    tx.execute(\
        "insert into z2ah5_jreviews_content (item_id, jr_retailer, affiliate_item_id, jr_description, date_created, date_modified) "
        "values (%s, %s, %s, %s, %s, %s)",
        (
            item['id'],
            item['retailer'],
            item['affiliate_item_id'],
            item['desc'],
            datetime.datetime.now(),
            datetime.datetime.now()
            )
        )
    return tx.lastrowid

def update_affiliate_item(self, tx, item, current_affiliate_item): #used to be this - current_affiliate_item
    tx.execute(\
        "update z2ah5_jreviews_content set jr_description = %s, date_modified = %s where contentid = %s ",
        (
            item['desc'],
            datetime.datetime.now(),
            current_affiliate_item['id'], #used to be this -          current_affiliate_item['id']
            )               
        )

Answer 1

The error is due to the fact that the dict affiliate_item does not have an ‘id’ key when you call the update_affiliate_item function

instead of this

if affiliate_item:

try this:

if affiliate_item and affiliate_item.has_key('id'):