Python爬虫爬取上海黄金交易所历史交易数据

为什么

因为想做上海黄金的量化交易，又信不过网上那些忽悠的神乎其神的App。于是自己动手，丰衣足食。

如何做

首先要获取权威的交易数据，上海黄金交易所官网就有历年的交易数据。所以打算用熟悉的Python写个爬虫自动获取。

1. 工具准备

Python3.6 + requests + lxml + Json

2. 网站解析

首先找到上海黄金交易所每日行情页列表（首页 > 数据资讯 > 历史行情数据 > 每日行情），分析该列表每页显示10天
的数据列表，点开后才是每天每个交易合约的交易数据。并且每一页的URL采用参数方式进行定位，如：“sjzx/mrhqsj?p=2 ”
表示第二页。所以只需要一个简单循环就可以找到需要的页面。

其次要找到具体数据页面列表的Xpath，可以使用浏览器Chrome自带的开发者模式，找到需要的数据，直接点右键 Copy > Copy Xpath。

3. 上代码

# -*- coding: UTF-8 -*-
# 本模块从上海黄金交易所官网下载历史交易数据
# https://www.sge.com.cn/sjzx/mrhqsj

import os
import time
from Lib.Web import get_Html, get_list, get_List_xpath, add_host
from Lib.os import save_list, save_list_A, save_list_B, makdir, BASE_PATH

headers = {
    ‘Referer‘: ‘https://www.sge.com.cn/sjzx/mrhqsj‘,
    ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36‘,
}


def get_table(title, url, headers):
    table_xpath = ‘//div[@class="content"]/table/tbody/tr/td[1]/text()‘
    html = get_Html(url, headers)
    doc = get_list(html, table_xpath)
    tab = []
    had = []
    n = len(doc)
    for r in range(1, n+1):
        table_xpath = ‘//div[@class="content"]/table/tbody/tr[%d]/td/text()‘ % r
        d = get_list(html, table_xpath)
        if r == 1:
            for i in d:
                had.append(str(i).replace(‘\t‘, ‘‘).replace(
                    ‘\n‘, ‘‘).replace(‘\r‘, ‘‘))
        else:
            row = {}
            row[‘交易日期‘] = title
            try:
                for i in range(len(d)):
                    row[had[i]] = str(d[i]).replace(
                        ‘\t‘, ‘‘).replace(‘\n‘, ‘‘).replace(‘\r‘, ‘‘)
            except Exception as e:
                pass
            tab.append(row)
    return tab


if __name__ == "__main__":

    # 获得下载链接
    for r in range(1, 201):
        url = ‘https://www.sge.com.cn/sjzx/mrhqsj?p=%d‘ % r
        filename = ‘list_%d.txt‘ % r
        cache_dir = "goldlist"

        html = get_Html(url, headers)

        if not os.path.exists(os.path.join(BASE_PATH, cache_dir)):
            makdir(os.path.join(BASE_PATH, cache_dir))

        filename = os.path.join(BASE_PATH, cache_dir, filename)
        if os.path.exists(filename):
            print("跳过：%s" % filename)
            continue

        a = ‘/html/body/div[6]/div/div[2]/div[2]/div[2]/ul/li/a/span[2]/text()‘
        b = ‘/html/body/div[6]/div/div[2]/div[2]/div[2]/ul/li/a/@href‘
        lst = get_List_xpath(html, a, b)
        for item in lst:
            lst[item] = add_host(url, lst[item])

        save_list_A(filename, lst)

        print(‘获取历史行情第%d页‘ % r)

        time.sleep(3)

    # 下载行情数据
    for r in range(1, 201):
        url = ‘https://www.sge.com.cn/sjzx/mrhqsj?p=%d‘ % r
        filename = ‘list_%d.txt‘ % r
        cache_dir = "goldlist"
        filename = os.path.join(BASE_PATH, cache_dir, filename)
        if os.path.exists(filename):
            with open(filename, ‘r‘, encoding=‘utf-8‘) as f:
                line = f.readline()
                item, url = line.split(‘\t‘)
                filename = os.path.join(BASE_PATH, cache_dir, "%s.txt" % item)

                if os.path.exists(filename):
                    print("跳过：%s" % filename)
                    continue

                doc = get_table(item, str(url).replace(‘\n‘, ‘‘), headers)
                save_list_B(filename, doc)

                print("保存：%s" % filename)

                time.sleep(3)

其中使用到我自己为了方便建立的库函数

def get_host(url):
    """ 返回域名，如：https://www.baidu.com """
    ul = urlparse(url)
    return ul.scheme + ‘://‘ + ul.hostname


def add_host(url, path):
    return get_host(url) + path


def get_Html(url, headers, cookies=None, params=None):
    """ 返回网页内容 """
    if cookies:
        r = requests.get(url=url, headers=headers, cookies=cookies)
    else:
        r = requests.get(url=url, headers=headers)
    r.encoding = "utf-8"
    return etree.HTML(r.text)


def get_list(html, xpath):
    """ 返回指定位置的列表 """
    return html.xpath(xpath)


def save_list_B(filename, list):
    with open(filename, ‘w‘, encoding=‘utf-8‘) as f:
        f.writelines(json.dumps(list,ensure_ascii=False))

Python爬虫爬取上海黄金交易所历史交易数据

原文：https://www.cnblogs.com/jeckun/p/14209477.html