From 625ae8fd198cf73406a08d950e124404dbbf4ec0 Mon Sep 17 00:00:00 2001 From: dennis Date: Sun, 26 Oct 2025 01:20:40 +0800 Subject: [PATCH] Initial commit --- README.md | 278 +++++++++++++++++++++++++++++++++++++++++++ appconfig/config.go | 48 ++++++++ config/config.yaml | 15 +++ config/db.yaml | 7 ++ config/longport.yaml | 10 ++ dbconn/dbconn.go | 78 ++++++++++++ go.mod | 24 ++++ go.sum | 55 +++++++++ main.go | 259 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 774 insertions(+) create mode 100644 README.md create mode 100644 appconfig/config.go create mode 100644 config/config.yaml create mode 100644 config/db.yaml create mode 100644 config/longport.yaml create mode 100644 dbconn/dbconn.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..413e508 --- /dev/null +++ b/README.md @@ -0,0 +1,278 @@ +# getk · 美股K线数据抓取 / US Stock K-Line Fetcher + +

+ 简体中文 · English +

+ +

+ Go + PostgreSQL + LongPort OpenAPI + Platform +

+ +--- + +## 简体中文 + +一个基于长桥证券接口（LongPort OpenAPI）的抓取工具，按配置批量抓取指定股票在指定日期的历史 K 线并写入 PostgreSQL。支持多只股票、多日期、去重插入、进度显示。 + +### 功能特性 +- 批量抓取指定标的的历史 K 线（分钟级） +- 多日期批处理，自动去重（`ON CONFLICT DO NOTHING`） +- 自动根据标的生成数据表（如 `AAPL.US` → `aapl`） +- 支持 `go run` 与二进制部署两种运行方式 + +### 环境要求 +- Go 1.20+（建议 1.21 或更高） +- PostgreSQL 12+ +- LongPort OpenAPI 凭证（App Key、App Secret、Access Token） + +### 如果没有 LongPort OpenAPI 凭证？ + +> 必须要在长桥证券开户，才能够开通 LongPort OpenAPI。 + +（一）长桥证券香港 +开户链接：[https://app.longbridgehk.com/ac/oa?account_channel=lb&channel=HB100036&invite-code=1OZ6MT](https://app.longbridgehk.com/ac/oa?account_channel=lb&channel=HB100036&invite-code=1OZ6MT) + +优势：配合香港银行卡出入无损，账户功能更加完善 +福利： +1. 入金等值 1W 港币得港股终身免佣金； +2. 入金 2 万港币或等值美金得 400 港币股票现金卡 + 5 美元现金卡 + 5% 收益货币基金购买资格； +3. 支持日内 10 倍融资，新加坡不支持。 + +（二）长桥证券新加坡 +开户链接：[https://activity.lbmkt.ing/pages/longbridge/7415/index.html?app_id=longbridge&org_id=1&account_channel=lb&lang=zh-CN&channel=HB100036&invite-code=1OZ6MT](https://activity.lbmkt.ing/pages/longbridge/7415/index.html?app_id=longbridge&org_id=1&account_channel=lb&lang=zh-CN&channel=HB100036&invite-code=1OZ6MT) + +优势：配合新加坡银行卡新币出入无损，有护照即可办理； +福利： +1. 入金 1800 新币可得港美股终生免佣； +2. 入金满 3400 新币得 72 新币股票现金卡 + 5 美元期权现金卡； +另外：用我链接开户的，再赠送 100 HKD 现金或者 18 新币。 + +- 开户成功后注册 LongPort OpenAPI 账号：[https://open.longportapp.com/](https://open.longportapp.com/) +- 创建应用并获取凭证（App Key、App Secret、Access Token） +- 配置 `config/longport.yaml` 或环境变量 + +### 快速运行（Windows 示例） +```powershell +# 直接运行（读取项目根的 config/） +go run .\main.go + +# 或构建后运行（读取 exe 同级的 config/） +go build -o getk.exe . +.\getk.exe +``` + +### 配置文件示例 +#### 1) `config/config.yaml` +```yaml +symbols: + - "AAPL.US" + - "TSLA.US" + - "NVDA.US" + +dates: + - "2025-10-15" + - "2025-10-16" + - "2025-10-17" + +settings: + period: "OneMinute" # 可选：OneMinute, FiveMinute, FifteenMinute, ThirtyMinute + adjust_type: "No" # 可选：No, ForwardAdjust +``` + +#### 2) `config/db.yaml` +```yaml +db: + host: "127.0.0.1" + port: 5432 + user: "postgres" + password: "your_password" + name: "your_database" + sslmode: "disable" +``` + +#### 3) `config/longport.yaml` +```yaml +longport: + app_key: "xxxxx" + app_secret: "xxxxx" + access_token: "xxxxx" +``` +也可用环境变量（PowerShell）： +```powershell +setx LONGPORT_APP_KEY "your-app-key" +setx LONGPORT_APP_SECRET "your-app-secret" +setx LONGPORT_ACCESS_TOKEN "your-access-token" +``` + +### 数据库表结构与写入规则 +- 表名：取股票代码点号前部分并转小写。例如 `AAPL.US` → `aapl`。 +- 去重：以 `timestamp` 为唯一键，`ON CONFLICT (timestamp) DO NOTHING` 跳过重复。 + +示例表结构（以 `aapl` 为例）： +```sql +CREATE TABLE IF NOT EXISTS aapl ( + timestamp TIMESTAMPTZ PRIMARY KEY, + open DOUBLE PRECISION, + close DOUBLE PRECISION, + high DOUBLE PRECISION, + low DOUBLE PRECISION, + volume BIGINT, + turnover DOUBLE PRECISION +); +``` + +### 常见问题 +- 读取配置失败：确认实际生效的配置目录内存在 `config.yaml / db.yaml / longport.yaml`。 +- LongPort 鉴权错误：检查 `app_key/app_secret/access_token` 是否有效，并确保账户具有行情权限。 + +### LongPort 接口频率限制 +- 每个账号同时只能建立一个长连接，最多同时订阅 500 个标的。 +- 1 秒内不超过 10 次调用，并发请求数不超过 5。 +- OpenAPI SDK 内部已经做了有效的频率控制： + - 行情类：`QuoteContext` 下的接口，SDK 会按照服务端频率限制自动控制；请求过快时会自动延迟。 + - 交易类：`TradeContext` 下的接口，SDK 没有做限制，需要用户自行处理。 + +### LongPort 使用费用 +LongPort 不针对接口服务额外收取开通或使用费用，只需开通 LongPort 账户及 OpenAPI 服务权限后即可免费使用。实际交易费率请咨询您开通证券账户的券商。 + +### 说明 +- 本项目使用 `github.com/longportapp/openapi-go/quote` 拉取历史 K 线。 +- 路径解析已优化以兼容 `go run` 与二进制部署场景。 + +--- + +## English + +A data fetcher powered by LongPort OpenAPI. It batches historical K-line (candlestick) data for specified symbols and dates, and writes into PostgreSQL. Supports multiple symbols, multiple dates, de-dup insert, and progress display. + +### Features +- Batch fetch minute-level historical K-line for selected symbols +- Multi-date processing with de-dup (`ON CONFLICT DO NOTHING`) +- Auto-create table per symbol (e.g. `AAPL.US` → `aapl`) +- Works with both `go run` and compiled binary deployments + +### Requirements +- Go 1.20+ (recommend 1.21+) +- PostgreSQL 12+ +- LongPort OpenAPI credentials (App Key, App Secret, Access Token) + +### Don’t have LongPort OpenAPI credentials? + +> LongBridge currently does not support customers from the United States and Canada due to licensing. You need a LongBridge brokerage account to enable OpenAPI. + +(1) LongBridge Hong Kong +Signup: [https://app.longbridgehk.com/ac/oa?account_channel=lb&channel=HB100036&invite-code=1OZ6MT](https://app.longbridgehk.com/ac/oa?account_channel=lb&channel=HB100036&invite-code=1OZ6MT) + +Pros: seamless HKD in/out with HK bank cards; more comprehensive account features +Benefits: +1. HK stock lifetime commission-free with a 10,000 HKD deposit; +2. Deposit 20,000 HKD (or equivalent USD) to receive 400 HKD stock cash card + 5 USD cash card + eligibility for 5% yield money market fund; +3. Supports 10x intraday margin (not supported in Singapore). + +(2) LongBridge Singapore +Signup: [https://activity.lbmkt.ing/pages/longbridge/7415/index.html?app_id=longbridge&org_id=1&account_channel=lb&lang=zh-CN&channel=HB100036&invite-code=1OZ6MT](https://activity.lbmkt.ing/pages/longbridge/7415/index.html?app_id=longbridge&org_id=1&account_channel=lb&lang=zh-CN&channel=HB100036&invite-code=1OZ6MT) + +Pros: seamless SGD in/out with SG bank cards; passport is sufficient +Benefits: +1. Deposit 1800 SGD for lifetime commission-free US/HK stocks; +2. Deposit 3400 SGD for a 72 SGD stock cash card + 5 USD options cash card. +Additionally, using my link grants an extra 100 HKD or 18 SGD. + +- Then go to: [https://open.longportapp.com/](https://open.longportapp.com/) +- Create an app and obtain credentials (App Key, App Secret, Access Token) +- Configure via `config/longport.yaml` or environment variables + +### Quick Start (Windows) +```powershell +# Run directly (reads config/ at project root) +go run .\main.go + +# Or build then run (reads config/ next to the exe) +go build -o getk.exe . +.\getk.exe +``` + +### Configuration Examples +#### 1) `config/config.yaml` +```yaml +symbols: + - "AAPL.US" + - "TSLA.US" + - "NVDA.US" + +dates: + - "2025-10-15" + - "2025-10-16" + - "2025-10-17" + +settings: + period: "OneMinute" # OneMinute, FiveMinute, FifteenMinute, ThirtyMinute + adjust_type: "No" # No, ForwardAdjust +``` + +#### 2) `config/db.yaml` +```yaml +db: + host: "127.0.0.1" + port: 5432 + user: "postgres" + password: "your_password" + name: "your_database" + sslmode: "disable" +``` + +#### 3) `config/longport.yaml` +```yaml +longport: + app_key: "xxxxx" + app_secret: "xxxxx" + access_token: "xxxxx" +``` +Or use environment variables (PowerShell): +```powershell +setx LONGPORT_APP_KEY "your-app-key" +setx LONGPORT_APP_SECRET "your-app-secret" +setx LONGPORT_ACCESS_TOKEN "your-access-token" +``` + +### Table Schema & Insert Rules +- Table name: take the part before the dot and lowercase (e.g. `AAPL.US` → `aapl`). +- De-dup: `timestamp` as primary key; `ON CONFLICT (timestamp) DO NOTHING`. + +Example (for `aapl`): +```sql +CREATE TABLE IF NOT EXISTS aapl ( + timestamp TIMESTAMPTZ PRIMARY KEY, + open DOUBLE PRECISION, + close DOUBLE PRECISION, + high DOUBLE PRECISION, + low DOUBLE PRECISION, + volume BIGINT, + turnover DOUBLE PRECISION +); +``` + +### FAQ +- Config read failure: ensure `config.yaml / db.yaml / longport.yaml` exist in the effective config directory. +- LongPort auth error: verify `app_key/app_secret/access_token` validity and that your account has market data permissions. + +### LongPort Rate Limits +- One concurrent persistent connection per account; up to 500 subscribed symbols. +- ≤ 10 calls per second; ≤ 5 concurrent requests. +- SDK-side throttling: + - Market data (`QuoteContext`): SDK auto-controls request rate per server policy and delays if too fast. + - Trading (`TradeContext`): no SDK limit; you must implement your own control. + +### Fees +No extra fees for using OpenAPI. You only need a LongBridge account and OpenAPI permissions; actual trading fees depend on your broker. + +### Notes +- Uses `github.com/longportapp/openapi-go/quote` to fetch historical K-lines. +- Path resolution works for both `go run` and binary deployments. + +--- + +> Tip: Click “简体中文” or “English” at the top to jump between language sections. diff --git a/appconfig/config.go b/appconfig/config.go new file mode 100644 index 0000000..0f7ddc9 --- /dev/null +++ b/appconfig/config.go @@ -0,0 +1,48 @@ +package appconfig + +import ( + "fmt" + "io/ioutil" + "time" + + "gopkg.in/yaml.v3" +) + +// Config 应用配置结构 +type Config struct { + Symbols []string `yaml:"symbols"` + Dates []string `yaml:"dates"` + Settings struct { + Period string `yaml:"period"` + AdjustType string `yaml:"adjust_type"` + } `yaml:"settings"` +} + +// Load 从YAML文件加载配置 +func Load(filepath string) (*Config, error) { + data, err := ioutil.ReadFile(filepath) + if err != nil { + return nil, fmt.Errorf("读取配置文件失败: %v", err) + } + + var config Config + err = yaml.Unmarshal(data, &config) + if err != nil { + return nil, fmt.Errorf("解析配置文件失败: %v", err) + } + + return &config, nil +} + +// ParseDates 将字符串日期转换为time.Time切片 +func (c *Config) ParseDates() ([]time.Time, error) { + var dates []time.Time + for _, dateStr := range c.Dates { + date, err := time.Parse("2006-01-02", dateStr) + if err != nil { + return nil, fmt.Errorf("解析日期 %s 失败: %v", dateStr, err) + } + dates = append(dates, date) + } + return dates, nil +} \ No newline at end of file diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..9f8fc8f --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,15 @@ +# 股票配置文件 +symbols: + - "AAPL.US" + - "TSLA.US" + - "NVDA.US" + +# 查询日期列表 (格式: YYYY-MM-DD) +dates: + - "2025-10-24" + - "2025-10-23" + +# 查询设置 +settings: + period: "OneMinute" # 可选: OneMinute, FiveMinute, FifteenMinute, ThirtyMinute + adjust_type: "No" # 可选: No, ForwardAdjust \ No newline at end of file diff --git a/config/db.yaml b/config/db.yaml new file mode 100644 index 0000000..9b87bcc --- /dev/null +++ b/config/db.yaml @@ -0,0 +1,7 @@ +db: + host: "127.0.0.1" + port: 5432 + user: "" + password: "" + name: "candlesticks" #default database name + sslmode: "disable" \ No newline at end of file diff --git a/config/longport.yaml b/config/longport.yaml new file mode 100644 index 0000000..0f5a483 --- /dev/null +++ b/config/longport.yaml @@ -0,0 +1,10 @@ +longport: + app_key: "" + app_secret: "" + access_token: "" + # Optional overrides: + # quote_url: "" + # trade_url: "" + # enable_overnight: false + # language: 0 # 0: zh-CN, 1: zh-TW, 2: en + # region: "cn" \ No newline at end of file diff --git a/dbconn/dbconn.go b/dbconn/dbconn.go new file mode 100644 index 0000000..45b048d --- /dev/null +++ b/dbconn/dbconn.go @@ -0,0 +1,78 @@ +package dbconn + +import ( + "database/sql" + "fmt" + "os" + + _ "github.com/lib/pq" + "gopkg.in/yaml.v3" +) + +// Config 表示数据库连接配置 +// 该配置与 ./db.yaml 中的结构一致 +// +// db: +// host: "127.0.0.1" +// port: 5432 +// user: "postgres" +// password: "..." +// name: "..." +// sslmode: "disable" +// +// 注意：此包仅负责连接与DSN构建，不做表创建逻辑 + +type Config struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + User string `yaml:"user"` + Password string `yaml:"password"` + Name string `yaml:"name"` + SSLMode string `yaml:"sslmode"` +} + +type appConfig struct { + DB Config `yaml:"db"` +} + +// Load 从指定 YAML 文件加载数据库配置 +func Load(path string) (Config, error) { + var cfg Config + b, err := os.ReadFile(path) + if err != nil { + return cfg, err + } + var ac appConfig + if err := yaml.Unmarshal(b, &ac); err != nil { + return cfg, err + } + return ac.DB, nil +} + +// DSN 构建 Postgres 连接字符串 +func DSN(cfg Config) string { + return fmt.Sprintf( + "user=%s password=%s dbname=%s host=%s port=%d sslmode=%s", + cfg.User, + cfg.Password, + cfg.Name, + cfg.Host, + cfg.Port, + cfg.SSLMode, + ) +} + +// Open 根据配置建立数据库连接（不验证连接可用性） +func Open(cfg Config) (*sql.DB, error) { + dsn := DSN(cfg) + return sql.Open("postgres", dsn) +} + +// OpenFromFile 便捷函数：从文件加载配置并建立连接 +func OpenFromFile(path string) (*sql.DB, error) { + cfg, err := Load(path) + if err != nil { + return nil, err + } + return Open(cfg) +} \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0f4b0ad --- /dev/null +++ b/go.mod @@ -0,0 +1,24 @@ +module getk + +go 1.25.3 + +require ( + github.com/lib/pq v1.10.9 + github.com/longportapp/openapi-go v0.16.4 +) + +require ( + github.com/Allenxuxu/ringbuffer v0.0.11 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/jinzhu/copier v0.3.5 // indirect + github.com/joho/godotenv v1.4.0 // indirect + github.com/longportapp/openapi-protobufs/gen/go v0.5.0 // indirect + github.com/longportapp/openapi-protocol/go v0.4.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3b6633b --- /dev/null +++ b/go.sum @@ -0,0 +1,55 @@ +github.com/Allenxuxu/ringbuffer v0.0.11 h1:51J/QakUlldfRBeKFAy81PD0IunxOQehvoBG/EvWT7k= +github.com/Allenxuxu/ringbuffer v0.0.11/go.mod h1:F2Ela+/miJmKYwnXr3X0+spOmSEwL/iFAEzeUJ4SFMI= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d h1:wvStE9wLpws31NiWUx+38wny1msZ/tm+eL5xmm4Y7So= +github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d/go.mod h1:9XMFaCeRyW7fC9XJOWQ+NdAv8VLG7ys7l3x4ozEGLUQ= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/longbridgeapp/assert v0.1.0 h1:KkQlHUJSpuUFkUDjwBJgghFl31+wwSDHTq/WRrvLjko= +github.com/longbridgeapp/assert v0.1.0/go.mod h1:ew3umReliXtk1bBG4weVURxdvR0tsN+rCEfjnA4YfxI= +github.com/longportapp/openapi-go v0.16.4 h1:oJSrngU+cCfmq7fF0+cI7KD01ZfHKXGHk8vVJXsW+UY= +github.com/longportapp/openapi-go v0.16.4/go.mod h1:oqDLdYKPCTqGx3GhJGZbeWuciwUihYF1Y85zCQVwi8I= +github.com/longportapp/openapi-protobufs/gen/go v0.5.0 h1:mKnkhI/D2X8+BKPEqRLjqkRpyAoCqjh8mIkYwl2gRv4= +github.com/longportapp/openapi-protobufs/gen/go v0.5.0/go.mod h1:/chiEwEW4CnOVgKTaCf8rQUwes00Ku8q1CvRpOueWfo= +github.com/longportapp/openapi-protocol/go v0.4.1 h1:H+sFW7kK8g10AAD6ZTGiKd7YaV5kZGCsEhGfd9JVyFY= +github.com/longportapp/openapi-protocol/go v0.4.1/go.mod h1:reREZxZflXsxFQ/VR4Uca25pMHyJ3nCYy86tp1hXva4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..f8b30c1 --- /dev/null +++ b/main.go @@ -0,0 +1,259 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "time" + + "getk/appconfig" + "getk/dbconn" + + "github.com/longportapp/openapi-go/config" + "github.com/longportapp/openapi-go/quote" +) + +type Candlestick struct { + Symbol string + Timestamp time.Time + Open float64 + Close float64 + High float64 + Low float64 + Volume int64 + Turnover float64 +} + +// 移除 GetOrCreateTable，改为直接根据 symbol 推导表名 +func InsertCandlesticks(db *sql.DB, symbol string, candlesticks []Candlestick) error { + tableName := strings.Split(symbol, ".")[0] + tableName = strings.ToLower(tableName) + + // Prepare existence check and insert statements for efficiency + existsStmt, err := db.Prepare(fmt.Sprintf(`SELECT EXISTS(SELECT 1 FROM %s WHERE timestamp = $1)`, tableName)) + if err != nil { + return err + } + defer existsStmt.Close() + + insertStmt, err := db.Prepare(fmt.Sprintf( + `INSERT INTO %s (timestamp, open, close, high, low, volume, turnover) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (timestamp) DO NOTHING`, + tableName, + )) + if err != nil { + return err + } + defer insertStmt.Close() + + for _, c := range candlesticks { + var exists bool + if err := existsStmt.QueryRow(c.Timestamp).Scan(&exists); err != nil { + log.Printf("Existence check failed: %v\n", err) + continue + } + if exists { + // Skip duplicates + continue + } + + _, err := insertStmt.Exec(c.Timestamp, c.Open, c.Close, c.High, c.Low, c.Volume, c.Turnover) + if err != nil { + log.Printf("Insert failed: %v\n", err) + } + } + + return nil +} + +// 自动创建数据表（若不存在） +func EnsureTable(db *sql.DB, symbol string) error { + tableName := strings.Split(symbol, ".")[0] + tableName = strings.ToLower(tableName) + createSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + timestamp TIMESTAMPTZ PRIMARY KEY, + open DOUBLE PRECISION, + close DOUBLE PRECISION, + high DOUBLE PRECISION, + low DOUBLE PRECISION, + volume BIGINT, + turnover DOUBLE PRECISION + )`, tableName) + _, err := db.Exec(createSQL) + return err +} + +func main() { + // 加载应用配置 + appCfg, err := appconfig.Load(configPath("config.yaml")) + if err != nil { + log.Fatalf("读取应用配置失败: %v", err) + } + + // 解析日期 + dates, err := appCfg.ParseDates() + if err != nil { + log.Fatalf("解析日期失败: %v", err) + } + + // 通过独立包加载数据库配置并建立连接 + dbCfg, err := dbconn.Load(configPath("db.yaml")) + if err != nil { + log.Fatalf("读取数据库配置失败: %v", err) + } + db, err := dbconn.Open(dbCfg) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // 通过配置文件加载 Longbridge 配置 (YAML/TOML) + conf, err := config.New(config.WithFilePath(configPath("longport.yaml"))) + if err != nil { + log.Fatalf("加载配置文件失败: %v", err) + } + + qctx, err := quote.NewFromCfg(conf) + if err != nil { + log.Fatal(err) + } + defer qctx.Close() + + // 获取期间类型 + period := getPeriodFromConfig(appCfg.Settings.Period) + adjustType := getAdjustTypeFromConfig(appCfg.Settings.AdjustType) + + fmt.Printf("开始数据获取任务...\n") + fmt.Printf("配置的股票数量: %d\n", len(appCfg.Symbols)) + fmt.Printf("配置的日期数量: %d\n", len(dates)) + fmt.Printf("总任务数: %d\n\n", len(appCfg.Symbols)*len(dates)) + + totalTasks := len(appCfg.Symbols) * len(dates) + currentTask := 0 + + // 循环处理每个股票的每个日期 + for _, symbol := range appCfg.Symbols { + fmt.Printf("📈 开始处理股票: %s\n", symbol) + + // 确保目标表存在（自动创建） + if err := EnsureTable(db, symbol); err != nil { + fmt.Printf(" ❌ 创建表失败: %v\n", err) + continue + } + + for _, date := range dates { + currentTask++ + + // 显示进度 + fmt.Printf(" [%d/%d] 正在查询 %s 的 %s 数据...", + currentTask, totalTasks, symbol, date.Format("2006-01-02")) + + // 设置查询时间范围（一整天） + start := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC) + end := time.Date(date.Year(), date.Month(), date.Day(), 23, 59, 59, 0, time.UTC) + + // 获取K线数据 + candlesticks, err := qctx.HistoryCandlesticksByDate( + context.Background(), + symbol, + period, + adjustType, + &start, + &end, + ) + if err != nil { + fmt.Printf(" ❌ 失败: %v\n", err) + continue + } + + // 转换数据格式 + var records []Candlestick + for _, c := range candlesticks { + records = append(records, Candlestick{ + Symbol: symbol, + Timestamp: time.Unix(c.Timestamp, 0), + Open: c.Open.InexactFloat64(), + Close: c.Close.InexactFloat64(), + High: c.High.InexactFloat64(), + Low: c.Low.InexactFloat64(), + Volume: c.Volume, + Turnover: c.Turnover.InexactFloat64(), + }) + } + + // 插入数据 + err = InsertCandlesticks(db, symbol, records) + if err != nil { + fmt.Printf(" ❌ 数据库插入失败: %v\n", err) + continue + } + + fmt.Printf(" ✅ 完成 (获取 %d 条记录)\n", len(records)) + } + fmt.Printf("✅ 股票 %s 处理完成\n\n", symbol) + } + + fmt.Printf("🎉 所有数据已保存到PostgreSQL！\n") + fmt.Printf("总共处理了 %d 个任务\n", totalTasks) +} + +// getPeriodFromConfig 根据配置字符串返回对应的Period类型 +func getPeriodFromConfig(periodStr string) quote.Period { + switch periodStr { + case "OneMinute": + return quote.PeriodOneMinute + case "FiveMinute": + return quote.PeriodFiveMinute + case "FifteenMinute": + return quote.PeriodFifteenMinute + case "ThirtyMinute": + return quote.PeriodThirtyMinute + default: + return quote.PeriodOneMinute + } +} + +// getAdjustTypeFromConfig 根据配置字符串返回对应的AdjustType类型 +func getAdjustTypeFromConfig(adjustStr string) quote.AdjustType { + switch adjustStr { + case "No": + return quote.AdjustTypeNo + case "ForwardAdjust": + return quote.AdjustTypeForward + default: + return quote.AdjustTypeNo + } +} + +// configPath 返回与可执行文件同级的 config 目录下的文件路径 +func configPath(filename string) string { + // 1) 优先使用环境变量 CONFIG_DIR（可设置为绝对或相对目录） + if dir := os.Getenv("CONFIG_DIR"); dir != "" { + return filepath.Join(dir, filename) + } + + // 2) 优先尝试当前工作目录下的 config 目录（兼容 go run） + if wd, err := os.Getwd(); err == nil { + p := filepath.Join(wd, "config", filename) + if _, err := os.Stat(p); err == nil { + return p + } + } + + // 3) 回退到可执行文件所在目录的 config 目录（兼容已编译二进制的部署场景） + if exe, err := os.Executable(); err == nil { + dir := filepath.Dir(exe) + p := filepath.Join(dir, "config", filename) + if _, err := os.Stat(p); err == nil { + return p + } + } + + // 4) 最后回退到相对路径，交由后续读取时报错并提示 + return filepath.Join(".", "config", filename) +}