The script custom pipeline class saves the collected data to mongodb

From , 4 Years ago, written in Python, viewed 55 times.

# Standard Python library imports
 
# 3rd party modules
import pymongo
 
from scrapy import log
from scrapy.conf import settings
from scrapy.exceptions import DropItem
 
 
class MongoDBPipeline(object):
    def __init__(self):
        self.server = settings['MONGODB_SERVER']
        self.port = settings['MONGODB_PORT']
        self.db = settings['MONGODB_DB']
        self.col = settings['MONGODB_COLLECTION']
        connection = pymongo.Connection(self.server, self.port)
        db = connection[self.db]
        self.collection = db[self.col]
 
    def process_item(self, item, spider):
        err_msg = ''
        for field, data in item.items():
            if not data:
                err_msg += 'Missing %s of poem from %s\n' % (field, item['url'])
        if err_msg:
            raise DropItem(err_msg)
        self.collection.insert(dict(item))
        log.msg('Item written to MongoDB database %s/%s' % (self.db, self.col),
                level=log.DEBUG, spider=spider)
        return item
#//python/8390

Reply to "The script custom pipeline class saves the collected data to mongodb"

Here you can reply to the paste above

Author What's your name?

Title Give your paste a title.

Language What language is your paste written in?

Your paste Paste your paste here

# Standard Python library imports
 
# 3rd party modules
import pymongo
 
from scrapy import log
from scrapy.conf import settings
from scrapy.exceptions import DropItem
 
 
class MongoDBPipeline(object):
    def __init__(self):
        self.server = settings['MONGODB_SERVER']
        self.port = settings['MONGODB_PORT']
        self.db = settings['MONGODB_DB']
        self.col = settings['MONGODB_COLLECTION']
        connection = pymongo.Connection(self.server, self.port)
        db = connection[self.db]
        self.collection = db[self.col]
 
    def process_item(self, item, spider):
        err_msg = ''
        for field, data in item.items():
            if not data:
                err_msg += 'Missing %s of poem from %s\n' % (field, item['url'])
        if err_msg:
            raise DropItem(err_msg)
        self.collection.insert(dict(item))
        log.msg('Item written to MongoDB database %s/%s' % (self.db, self.col),
                level=log.DEBUG, spider=spider)
        return item
#//python/8390

Create Shorturl Create a shorter url that redirects to your paste?

Private Private paste aren't shown in recent listings.

Delete After When should we delete your paste?

Spam protection Type in the letters

https://burned.cc - Burn After Reading Website

Pastebin.vip

The script custom pipeline class saves the collected data to mongodb

Reply to "The script custom pipeline class saves the collected data to mongodb"