在Python3中,我們可以使用多種庫來保存爬取到的數據。以下是一些建議的庫和方法:
import sqlite3
# 創建數據庫連接
conn = sqlite3.connect('example.db')
cursor = conn.cursor()
# 創建表
cursor.execute('''CREATE TABLE IF NOT EXISTS data (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
url TEXT NOT NULL)''')
# 插入數據
def insert_data(title, url):
cursor.execute("INSERT INTO data (title, url) VALUES (?, ?)", (title, url))
conn.commit()
# 查詢數據
def query_data():
cursor.execute("SELECT * FROM data")
return cursor.fetchall()
# 更新數據
def update_data(id, title, url):
cursor.execute("UPDATE data SET title = ?, url = ? WHERE id = ?", (title, url, id))
conn.commit()
# 關閉數據庫連接
conn.close()
import json
data = [
{"title": "Title 1", "url": "https://example.com/1"},
{"title": "Title 2", "url": "https://example.com/2"},
]
# 保存為JSON文件
with open('data.json', 'w') as f:
json.dump(data, f)
import csv
data = [
{"title": "Title 1", "url": "https://example.com/1"},
{"title": "Title 2", "url": "https://example.com/2"},
]
# 保存為CSV文件
with open('data.csv', 'w', newline='', encoding='utf-8') as f:
writer = csv.DictWriter(f, fieldnames=["title", "url"])
writer.writeheader()
for row in data:
writer.writerow(row)
pip install pymongo
),然后連接到MongoDB數據庫,插入數據,查詢和更新數據。from pymongo import MongoClient
# 創建數據庫連接
client = MongoClient('mongodb://localhost:27017/')
db = client['example_db']
collection = db['data']
# 插入數據
def insert_data(title, url):
collection.insert_one({"title": title, "url": url})
# 查詢數據
def query_data():
return collection.find()
# 更新數據
def update_data(id, title, url):
collection.update_one({"_id": id}, {"$set": {"title": title, "url": url}})
# 關閉數據庫連接
client.close()
根據你的需求和數據類型,可以選擇合適的庫和方法來保存爬取到的數據。