3.5 Python操作MongoDB

我们使用库PyMongo来操作MongoDB，这个库封装了MongoDB操作的所有常见操作。可以在官网查看详细API。

1. 创建连接

import pymongo

# 创建MongoDB连接
# Making a Connection with MongoClient
client = pymongo.MongoClient(host='localhost', port=27017)

2. 获取/创建数据库、集合

# 获取/创建数据库
# Getting/Creating a Database
db = client.test
# db = client['test']

# 获取/创建集合
# Getting/Creating a collection
collection = db.students
# collection = db['students']

3. 插入文档数据

# 插入文档
# Documents, Data in MongoDB is represented (and stored) using JSON-style documents. 
# In PyMongo we use dictionaries to represent documents.
# As an example, the following dictionary might be used to represent a blog post:
student1 = {
    'id': '20170101',
    'name': 'Jordan',
    'age': 20,
    'gender': 'male'
}

students = [
    {
    'id': '20160101',
    'name': 'Marco',
    'age': 21,
    'gender': 'male'
    },
    {
    'id': '20160101',
    'name': 'Lilei',
    'age': 22,
    'gender': 'male'
    }
]

# Inserting a document to collection
result = collection.insert_one(student1)

# Inserting many documents to collection
result = collection.insert_many(students)

4. 查询文档数据

# 查询文档数据
# Getting a Single Document With find_one()
print(collection.find_one())
# Search document with key word for a single document
print(collection.find_one({'name':'Lilei'}))

# Querying for More Than One Document:
# To get more than a single document as the result of a query we use the find() method. 
# find() returns a Cursor instance, which allows us to iterate over all matching documents.
# For example, we can iterate over every document in the posts collection:
print(collection.find())
for item in collection.find():
    print(item)

# 范围查询
# MongoDB supports many different types of advanced queries. 
# As an example, lets perform a query where we limit results to the students who is older than 20, but also sort the results by name:
for item in collection.find({"age":{"$gt":20}}):
    print(item)

# 正则表达式查询
for item in collection.find({'name': {'$regex': '^J.*'}}):
    print(item)

上面的高级查询中的条件键值已经不是单纯的数字了，而是一个字典，其键名为比较符号 $gt，意思是大于，键值为 20，这样便可以查询出所有年龄大于 20 的数据。比较符号归纳如下表：

符号	含义	示例
$lt	小于	{'age': {'$lt': 20}}
$gt	大于	{'age': {'$gt': 20}}
$lte	小于等于	{'age': {'$lte': 20}}
$gte	大于等于	{'age': {'$gte': 20}}
$ne	不等于	{'age': {'$ne': 20}}
$in	在范围内	{'age': {'$in': [20, 23]}}
$nin	不在范围内	{'age': {'$nin': [20, 23]}}

上面的例子中使用了$regex 来指定正则匹配，^M.* 代表以 M 开头的正则表达式，这样就可以查询所有符合该正则的结果。将一些功能符号再归类如下：

符号	含义	示例	示例含义
$regex	匹配正则	{'name': {'$regex': '^M.*'}}	name 以 M开头
$exists	属性是否存在	{'name': {'$exists': True}}	name 属性存在
$type	类型判断	{'age': {'$type': 'int'}}	age 的类型为 int
$mod	数字模操作	{'age': {'$mod': [5, 0]}}	年龄模 5 余 0
$text	文本查询	{'$text': {'$search': 'Mike'}}	text 类型的属性中包含 Mike 字符串
$where	高级条件查询	{'$where': 'obj.fans_count == obj.follows_count'}	自身粉丝数等于关注数

这些操作的更详细用法在可以在 MongoDB 官方文档找到：https://docs.mongodb.com/manual/reference/operator/query/。

5. 计数

# 查找计数
# If we just want to know how many documents match a query we can perform a count() operation instead of a full query. 
# We can get a count of all of the documents in a collection
print(collection.find().count())
print(collection.find({'name':'Lilei'}).count())

6. 排序

# 排序
# 使用sort()函数进行排序，函数中可以指定参数: pymongo.ASCENDING 指定升序，或者pymongo.DESCENDING降序。
for item in collection.find({"age":{"$gt":20}}).sort('name',pymongo.ASCENDING):
    print(item)

7. 偏移与限制查询个数

# 偏移与限制查询个数
# skip(n):忽略查询出来的前n个结果，从n+1结果开始算起
for item in collection.find({"age":{"$gt":20}}).sort('name').skip(1):
    print(item)
# limit(n):只取查询出来的前n个结果
for item in collection.find({"age":{"$gt":20}}).sort('name').skip(1).limit(1):
    print(item)

8. 更新数据

# 更新数据
# 使用update_one()函数进行修改更新一条数据，其中需要使用 $set,$inc 操作符对数据进行更新
condition = {'name': 'Marco'}
student = collection.find_one(condition)
student['age'] = 26
result = collection.update_one(condition,{'$set':student})
print(result)
print(result.matched_count, result.modified_count)
# 使用update_many()函数进行修改更新多条数据，其中需要使用 $set,$inc 操作符对数据进行更新
condition = {'age': {'$gt': 20}}
result = collection.update_many(condition, {'$inc': {'age': 1}})
print(result)
print(result.matched_count, result.modified_count)

9. 删除

# 删除
# 使用remove(), delete_one(), delete_many()函数进行删除数据
result = collection.remove({'name': 'Kevin'})
print(result)
result = collection.delete_one({'name': 'Kevin'})
print(result,result.deleted_count)
result = collection.delete_many({'age': {'$lt': 25}})
print(result.deleted_count)

10. 案例实操:爬取猫眼电影存储到MongoDB

应用库：Requests发起请求获取响应数据, PyQuery解析响应数据获得想要的数据, pymongo保存数据到MongoDB数据库

import requests
import re
from pyquery import PyQuery as pq
import json
import urllib
from urllib import error
import pymongo

# 创建 一个MongoDB连接
client = pymongo.MongoClient('localhost', 27017)
# 获取mongodb数据库:maoyandb，没有就新建该数据库
db = client.maoyandb
# 获取该数据库的一个集合:leaderboard,没有就新建该集合
collection = db.leaderboard


def get_one_page(url):
    headers = {
    'Cookie': 'uuid_n_v=v1; uuid=6266F950913511E8BAE959978C7B0D0473005491BEDE4F03918D808909FBCD63; _csrf=ac998ee17cc0a0a19ead8ae7bad340ccd342180e2588ba8b326ee6d6b4deac25; _lxsdk_cuid=164d92c0d5a25-0e046d8f7973c-16386952-100200-164d92c0d5bc8; _lxsdk=6266F950913511E8BAE959978C7B0D0473005491BEDE4F03918D808909FBCD63; __mta=152329862.1532651901646.1532653173231.1532653961468.14; _lxsdk_s=164d92c0d5c-908-59-aa6%7C%7C56',
    'Host': 'maoyan.com',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
    proxies = {
        #换成自己的代理
        "http": "http://m00240582:[e-mail]:8080",
        "https": "http://m00240582:[e-mail]:8080",
    }
    try:
        response = requests.get(url,headers=headers,proxies=proxies)
        return response.text
    except error.HTTPError as err:
        print(err.reason, err.code, err.headers, sep='\n')
        return None
    except urllib.error.URLError as err:
        print(err.reason)
        return None

def parse_one_page(html):
    doc = pq(html)
    items = doc('dd').items()
    for item in items:
        yield{
            '排名': item.find('.board-index').text(),
            '电影': item.find('.image-link').attr('title'),
            '海报链接': item.find('.image-link .board-img').attr('data-src'),
            # 添加条件判断语句，增加程序的健壮性，只有当长度大于3时才从第三个开始获取
            '演员': item.find('.movie-item-info').find('p.star').text()[3:] if len(item.find('.movie-item-info').find('p.star').text())>3 else '',
            # 添加条件判断语句，增加程序的健壮性，只有当长度大于5时才能获取上映时间
            '上映时间': item.find('.movie-item-info').find('p.releasetime').text()[5:15] if len(item.find('.movie-item-info').find('p.star').text())>5 else '',
            # 通过正则表达式获取括号()里的字符串，且添加条件判断语句只有匹配到了才获取其中的字符串，
            '上映地点': re.findall(r'\((.*?)\)',item.find('.movie-item-info').find('p.releasetime').text(),re.S)[0] if len(re.findall(r'\((.*?)\)',item.find('.movie-item-info').find('p.releasetime').text(),re.S))>0 else '',
            # 将原有的字符串变成数字,将评分前半段去掉点号，小数位除以10
            '猫眼评分': int(item.find('p.score i.integer').text().strip('.'))+int(item.find('p.score i.fraction').text())/10
        }

def save_to_mongodn(data):
    # data就是一个json对象数据
    collection.insert_one(document=data)

def single_page(offset):
    url = 'http://maoyan.com/board/4?offset='+str(offset*10)
    html = get_one_page(url)
    for item in parse_one_page(html):
        print(item)
        save_to_mongodn(item)

if __name__ == '__main__':
    for page in range(10):
        single_page(page)
    condition = {'猫眼评分': {'$gt': 9.5}}
    print(collection.find(condition).count())
    for item in collection.find(condition):
        print(item)