Compare commits

...
17 Commits
Author SHA1 Message Date
Yun 6257edf1c7 添加获取错误的方法 2026-03-04 18:23:01 +08:00
Yun d3b39ac7be 优化response,改为小写,避免直接暴露 2026-03-04 18:20:31 +08:00
yun f15c882874 一些example 2026-03-01 23:10:09 +08:00
yun 3e96b1ada6 再次修改options的使用 2026-03-01 23:06:20 +08:00
yun d6e1229961 优化响应日志的打印 2026-03-01 22:28:59 +08:00
yun 3f51fb39cb 优化日志打印 2026-03-01 19:03:31 +08:00
yun 0427f4a844 修改日志接口 2026-03-01 13:26:38 +08:00
yun 660f3dee83 更新包的版本 2025-12-29 22:51:54 +08:00
yun 0931539e1c 调整封装的方法 2025-12-28 15:14:05 +08:00
yun 8bca4cc013 更新 2025-09-13 22:23:29 +08:00
yun 6a9f388bb6 调整请求 2025-09-13 22:20:06 +08:00
yun 14bb42c131 Merge remote-tracking branch 'github/master' into dev 2025-09-13 17:50:24 +08:00
Yun cd521e4837 添加支持单个header参数 2025-08-21 11:08:13 +08:00
Yun aa0e094851 修改流式响应的超时时间 2025-01-07 13:53:36 +08:00
Yun 566d0b7b51 添加代理访问测试 2024-12-12 10:16:15 +08:00
Yun 6387f372c4 优化socks5代理 2024-12-11 19:24:49 +08:00
yun-ink 2cf94d88ab test 2024-04-21 01:06:59 +00:00
13 changed files with 986 additions and 441 deletions
+17
View File
@@ -0,0 +1,17 @@
package curlx
import "errors"
type UserAgent string
const (
UserAgentChrome UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
UserAgentFirefox UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 "
UserAgentIE UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; "
UserAgentEdge UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
UserAgentWechat UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
)
var (
ErrStatusNotOK error = errors.New("Status not ok")
)
+96 -150
View File
@@ -2,10 +2,8 @@ package curlx
import (
"bufio"
"compress/gzip"
"context"
"fmt"
"io"
"errors"
"net"
"net/http"
"net/url"
@@ -22,7 +20,7 @@ import (
// type DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
type Curlx struct {
opts clientOptions
opts ClientOptions
transport *http.Transport
}
@@ -33,31 +31,13 @@ func NewCurlx(opts ...Option) *Curlx {
}
transport := &http.Transport{
// Dial: func(netw, addr string) (net.Conn, error) {
// // 这里指定域名访问的IP
// // if addr == "api.hk.blueoceanpay.com:443" {
// // addr = "47.56.200.21:443"
// // }
// conn, err := net.DialTimeout(netw, addr, time.Second*time.Duration(timeOut)) // 设置建立连接超时
// if err != nil {
// return nil, err
// }
// // conn.RemoteAddr().String()
// conn.SetDeadline(time.Now().Add(time.Second * time.Duration(timeOut))) // 设置发送接收数据超时
// return conn, nil
// },
// DialContext: (&net.Dialer{
// Timeout: 3 * time.Second, // 建立TCP链接的超时时间
// KeepAlive: 30 * time.Second, // TCP keepalive超时时间
// }).DialContext,
// TLSHandshakeTimeout: time.Second * 10, // TLS握手超时
// ResponseHeaderTimeout: time.Second * 10, // 接收响应头的超时时间
// ExpectContinueTimeout: time.Second * 10, // 发送请求头超时时间 100-continue状态码超时时间
DisableKeepAlives: false, // 短连接(默认是使用长连接,连接过多时会造成服务器拒绝服务问题)
MaxIdleConns: 0, // 所有host的连接池最大连接数量,默认无穷大
MaxIdleConnsPerHost: 5, // 每个host的连接池最大空闲连接收,默认2
MaxConnsPerHost: 0, // 每个host的最大连接数量
IdleConnTimeout: time.Second * 2, // 空闲连接超时关闭的时间
DisableKeepAlives: false, // 启用keep-alive连接复用
MaxIdleConns: defaultOpts.MaxIdleConns,
MaxIdleConnsPerHost: defaultOpts.MaxIdleConnsPerHost,
MaxConnsPerHost: defaultOpts.MaxConnsPerHost,
IdleConnTimeout: defaultOpts.IdleConnTimeout,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
}
if defaultOpts.InsecureSkipVerify {
@@ -79,10 +59,14 @@ func (c *Curlx) WithProxySocks5(address string) error {
baseDialer := &net.Dialer{
// Timeout: 180 * time.Second,
// KeepAlive: 180 * time.Second,
Resolver: &net.Resolver{
PreferGo: true,
Dial: c.transport.DialContext,
},
}
dialSocksProxy, err := proxy.SOCKS5("tcp", address, nil, baseDialer)
if err != nil {
fmt.Println("proxy.SOCKS5 err", err)
c.opts.Logger.Errorf(context.Background(), "proxy.SOCKS5 err: %v", err)
return err
}
dialContext := (baseDialer).DialContext
@@ -94,12 +78,13 @@ func (c *Curlx) WithProxySocks5(address string) error {
}
/**
* 使用HTTP代理
* 使用HTTP/HTTPS代理
* @param proxyAddr "https://proxyserver:port"
*/
func (c *Curlx) WithProxyHttp(proxyAddr string) error {
proxy, err := url.Parse(proxyAddr)
if err != nil {
c.opts.Logger.Errorf(context.Background(), "proxy.HTTP/HTTPS err: %v", err)
return err
}
c.transport.Proxy = http.ProxyURL(proxy)
@@ -118,154 +103,110 @@ func (c *Curlx) WithAddress(ctx context.Context, addr string) {
/**
* 简单请求
*/
func (c *Curlx) Send(ctx context.Context, p ...Param) (res []byte, httpcode int, err error) {
_, response, err := c.SendExec(ctx, p...)
func (c *Curlx) Send(ctx context.Context, p ...Param) (res []byte, err error) {
resp := c.exec(ctx, p...)
if resp.err != nil {
return nil, resp.err
}
defer resp.Close() // 处理完关闭
status := resp.GetStatusCode()
if status != 200 {
c.opts.Logger.Errorf(ctx, "curlx.Send status not OK: %d", status)
return nil, ErrStatusNotOK
}
body, err := resp.GetBody()
if err != nil {
return nil, -1, err
c.opts.Logger.Errorf(ctx, "curlx.Send getBody err:%v", err)
return nil, err
}
defer response.Body.Close() // 处理完关闭
// stdout := os.Stdout // 将结果定位到标准输出,也可以直接打印出来,或定位到其他地方进行相应处理
// _, err = io.Copy(stdout, response.Body) // 将第二个参数拷贝到第一个参数,直到第二参数到达EOF或发生错误,返回拷贝的值
status := response.StatusCode // 获取状态码,正常是200
var body []byte
// switch response.Header.Get("Content-Encoding") {
// case "gzip":
// reader, err := gzip.NewReader(response.Body)
// if err != nil {
// return nil, status, err
// }
// for {
// buf := make([]byte, 1024)
// n, err := reader.Read(buf)
// if err != nil && err != io.EOF {
// panic(err)
// }
// if n == 0 {
// break
// }
// body = append(body, buf...)
// }
// default:
// body, _ = io.ReadAll(response.Body)
// }
if response.Header.Get("Content-Encoding") == "gzip" {
reader, err := gzip.NewReader(response.Body)
if err != nil {
return nil, response.StatusCode, err
}
body, err = io.ReadAll(reader)
if err != nil {
return nil, response.StatusCode, err
}
defer reader.Close()
} else {
body, err = io.ReadAll(response.Body)
if err != nil {
return nil, response.StatusCode, err
// 打印日志时截取前指定长度,避免日志过大
bodyLog := []rune(string(body))
if len(bodyLog) > c.opts.LoggerLength {
bodyLog = bodyLog[:c.opts.LoggerLength]
}
c.opts.Logger.Infof(ctx, "curlx.Send response body:%s", string(bodyLog))
return body, nil
}
c.opts.Logger.Infof(ctx, "curlx.Send body:%s", string(body))
return body, status, nil
// PostJson 发送JSON数据
func (l *Curlx) PostJson(ctx context.Context, url string, jsonStr string) ([]byte, error) {
return l.Send(ctx,
SetParamsUrl(url),
SetParamsBody([]byte(jsonStr)),
SetParamsContentType(ContentTypeJson),
SetParamsMethod(MethodPost),
)
}
func (c *Curlx) SendWithResponee(ctx context.Context, ps ...Param) Response {
r := Response{}
req, resp, err := c.SendExec(ctx, ps...)
r.req = req
r.resp = resp
if err != nil {
r.err = err
return r
}
var body []byte
// switch resp.Header.Get("Content-Encoding") {
// case "gzip":
// reader, err := gzip.NewReader(resp.Body)
// if err != nil {
// r.err = err
// return r
// }
// for {
// buf := make([]byte, 1024)
// n, err := reader.Read(buf)
// if err != nil && err != io.EOF {
// panic(err)
// }
// if n == 0 {
// break
// }
// // 读取n个字节
// body = append(body, buf[:n]...)
// // body = append(body, buf...)
// }
// default:
// body, _ = io.ReadAll(resp.Body)
// }
if resp.Header.Get("Content-Encoding") == "gzip" {
reader, err := gzip.NewReader(resp.Body)
if err != nil {
r.err = err
return r
}
body, err = io.ReadAll(reader)
if err != nil {
r.err = err
return r
}
defer reader.Close()
} else {
body, err = io.ReadAll(resp.Body)
if err != nil {
r.err = err
return r
}
// Get 简单GET请求
func (l *Curlx) Get(ctx context.Context, url string) ([]byte, error) {
return l.Send(ctx,
SetParamsUrl(url),
SetParamsMethod(MethodGet),
)
}
r.body = body
c.opts.Logger.Infof(ctx, "curlx.Send body:%s", string(body))
return r
func (c *Curlx) SendWithResponse(ctx context.Context, ps ...Param) Response {
return c.exec(ctx, ps...)
}
/**
* 执行发送
* 注意:外部使用需要加这一句 defer response.Body.Close()
*/
func (c *Curlx) SendExec(ctx context.Context, ps ...Param) (req *http.Request, resp *http.Response, err error) {
func (c *Curlx) exec(ctx context.Context, ps ...Param) Response {
resp := Response{}
client := &http.Client{
Timeout: c.opts.TimeOut, // 整个请求的超时时间 设置该条连接的超时
Transport: c.transport, //
}
// 在http.Client中添加CheckRedirect函数 实现重定向控制
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 { // 限制重定向次数
return errors.New("stopped after 10 redirects")
}
return nil
}
p := defaultParams()
for _, param := range ps {
param(&p)
}
c.opts.Logger.Infof(ctx, "curlx.sendExec params:%+v", p)
err = p.parseMethod()
// 截取Body前指定长度输出,避免日志过大
bodyLog := []rune(string(p.Body))
if len(bodyLog) > c.opts.LoggerLength {
bodyLog = bodyLog[:c.opts.LoggerLength]
}
c.opts.Logger.Infof(ctx, "curlx.sendExec params url:%s method:%s contentType:%s body:%s headers:%+v cookies:%+v", p.Url, p.Method, p.ContentType, string(bodyLog), p.Headers, p.Cookies)
err := p.parseMethod()
if err != nil {
return nil, nil, err
c.opts.Logger.Errorf(ctx, "curlx.sendExec parseMethod err:%v", err)
resp.err = err
return resp
}
// 判断和处理url
err = p.parseUrl()
if err != nil {
c.opts.Logger.Errorf(ctx, "curlx.sendExec parseUrl err:%v", err)
return nil, nil, err
resp.err = err
return resp
}
// 处理参数
reqParams, err := p.parseParams()
if err != nil {
c.opts.Logger.Errorf(ctx, "curlx.sendExec parseParams err:%v", err)
return nil, nil, err
resp.err = err
return resp
}
// 初始化句柄
@@ -276,9 +217,13 @@ func (c *Curlx) SendExec(ctx context.Context, ps ...Param) (req *http.Request, r
)
if err != nil {
c.opts.Logger.Errorf(ctx, "curlx.sendExec NewRequest err:%v", err)
return nil, nil, err
resp.err = err
return resp
}
c.opts.Logger.Infof(ctx, "curlx.sendExec request:%+v", request)
resp.request = request
// 这里指定要访问的HOST,到时候服务器获取主机是获取到这个
// request.Host = "api.hk.blueoceantech.co"
@@ -295,10 +240,12 @@ func (c *Curlx) SendExec(ctx context.Context, ps ...Param) (req *http.Request, r
response, err := client.Do(request)
if err != nil {
c.opts.Logger.Errorf(ctx, "curlx.sendExec client.Do err:%v", err)
return nil, nil, err
resp.err = err
return resp
}
// response.StatusCode
return request, response, nil
resp.response = response
return resp
}
/**
@@ -311,16 +258,15 @@ func (c *Curlx) SendStream(ctx context.Context, ps ...Param) (<-chan string, err
go func() {
defer close(data)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
ctx, cancel := context.WithTimeout(context.Background(), c.opts.TimeOut)
defer cancel()
_, response, err := c.SendExec(ctx, ps...)
if err != nil {
response := c.exec(ctx, ps...)
if response.err != nil {
return
}
defer response.Body.Close() // 处理完关闭
scanner := bufio.NewScanner(response.Body)
defer response.Close() // 处理完关闭
scanner := bufio.NewScanner(response.response.Body)
for scanner.Scan() {
text := scanner.Text()
if text == "" {
+19 -7
View File
@@ -2,18 +2,20 @@ package curlx
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"testing"
)
func TestGet(t *testing.T) {
resp, code, err := NewCurlx().Send(context.Background(),
resp, err := NewCurlx().Send(context.Background(),
SetParamsUrl("https://www.baidu.com"),
SetParamsMethod(MethodGet),
)
t.Log(resp, code, err)
t.Log(string(resp), err)
}
@@ -36,15 +38,25 @@ func TestForm(t *testing.T) {
},
}
by, _ := json.Marshal(s)
p := ClientParams{
Url: "http://tech-dev.sealmoo.com/api/material/upload",
Method: "POST",
Body: s,
Headers: map[string]interface{}{
"Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0ZW5hbnRfaWQiOjAsImNsaWVudF9pZCI6MCwidXNlcl9pZCI6MSwiZXhwIjoxNzAxMzk3NzkxfQ.9_uJ6y8I4JZTwgSenwHC_01nddLuI4zmgpyPhn5M6j8",
Body: by,
Headers: http.Header{
"Content-Type": []string{"application/json"},
"Authorization": []string{"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0ZW5hbnRfaWQiOjAsImNsaWVudF9pZCI6MCwidXNlcl9pZCI6MSwiZXhwIjoxNzAxMzk3NzkxfQ.9_uJ6y8I4JZTwgSenwHC_01nddLuI4zmgpyPhn5M6j8"},
},
ContentType: ContentTypeForm,
}
resp, code, err := NewCurlx().Send(context.Background(), SetParamsAll(p))
fmt.Println(resp, code, err)
resp, err := NewCurlx().Send(context.Background(), SetParamsAll(p))
fmt.Println(resp, err)
}
func TestProxy(t *testing.T) {
c := NewCurlx()
c.WithProxySocks5("127.0.0.1:1080")
res, err := c.Send(context.Background(), SetParamsUrl("https://www.google.com"), SetParamsMethod(MethodGet))
t.Log(string(res), err)
}
+199
View File
@@ -0,0 +1,199 @@
# HTTP连接复用最佳实践
## 什么是连接复用?
HTTP连接复用(Connection Reuse)是指在同一个HTTP客户端实例中,对相同目标主机的多个请求复用已建立的TCP连接,而不是为每个请求都创建新的连接。这可以显著提高性能并减少资源消耗。
## 为什么需要连接复用?
1. **性能提升**:避免重复的TCP三次握手和TLS握手
2. **资源节约**:减少系统文件描述符和内存使用
3. **降低延迟**:复用已建立的连接减少连接建立时间
4. **服务器友好**:减少服务器连接压力
## curlx中的连接复用配置
### 基本配置参数
```go
client := NewCurlx(
// 连接池大小配置
WithMaxIdleConns(100), // 总空闲连接数上限
WithMaxIdleConnsPerHost(10), // 每个主机的空闲连接数
WithMaxConnsPerHost(50), // 每个主机的最大连接数
WithIdleConnTimeout(90*time.Second), // 空闲连接超时时间
// 其他优化配置
SetOptionTimeOut(30*time.Second),
)
```
### 参数详解
| 参数 | 默认值 | 说明 |
|------|--------|------|
| `MaxIdleConns` | 100 | 连接池中保持的最大空闲连接总数 |
| `MaxIdleConnsPerHost` | 10 | 对每个主机保持的最大空闲连接数 |
| `MaxConnsPerHost` | 50 | 对每个主机允许的最大并发连接数 |
| `IdleConnTimeout` | 90s | 空闲连接的超时时间 |
## 最佳实践
### 1. 正确使用单例模式
```go
// ❌ 错误做法:每次请求都创建新客户端
func badExample() {
for i := 0; i < 100; i++ {
client := NewCurlx() // 每次都新建,无法复用连接
client.Get(context.Background(), "https://example.com")
}
}
// ✅ 正确做法:复用客户端实例
func goodExample() {
client := NewCurlx( // 只创建一次
WithMaxIdleConns(50),
WithMaxIdleConnsPerHost(5),
)
for i := 0; i < 100; i++ {
client.Get(context.Background(), "https://example.com") // 复用连接
}
}
```
### 2. 合理设置连接池大小
```go
// 根据应用场景调整配置
func getConfigForScenario(scenario string) []Option {
switch scenario {
case "high_concurrency":
return []Option{
WithMaxIdleConns(200),
WithMaxIdleConnsPerHost(20),
WithMaxConnsPerHost(100),
}
case "low_resource":
return []Option{
WithMaxIdleConns(20),
WithMaxIdleConnsPerHost(2),
WithMaxConnsPerHost(10),
}
default:
return []Option{
WithMaxIdleConns(100),
WithMaxIdleConnsPerHost(10),
WithMaxConnsPerHost(50),
}
}
}
```
### 3. 监控连接池状态
```go
func monitorConnectionPool(client *Curlx) {
manager := &ConnectionPoolManager{
client: client,
transport: client.transport,
}
// 定期检查连接池状态
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for range ticker.C {
manager.PrintPoolStats()
}
}
```
## 性能对比测试
```go
func BenchmarkConnectionReuse(b *testing.B) {
client := NewCurlx(
WithMaxIdleConns(50),
WithMaxIdleConnsPerHost(10),
)
b.Run("with_reuse", func(b *testing.B) {
for i := 0; i < b.N; i++ {
client.Get(context.Background(), "https://httpbin.org/get")
}
})
}
func BenchmarkWithoutReuse(b *testing.B) {
b.Run("without_reuse", func(b *testing.B) {
for i := 0; i < b.N; i++ {
client := NewCurlx() // 每次新建客户端
client.Get(context.Background(), "https://httpbin.org/get")
}
})
}
```
## 常见问题解答
### Q: 连接池满了怎么办?
A: 当连接池满时,新的请求会等待空闲连接。可以通过增加`MaxConnsPerHost`来缓解。
### Q: 如何清理空闲连接?
A: 空闲连接会在`IdleConnTimeout`后自动关闭,也可以手动调用`transport.CloseIdleConnections()`
### Q: 不同主机的连接是否共享?
A: 不同主机的连接是隔离的,每个主机维护自己的连接池。
### Q: HTTPS连接也能复用吗?
A: 是的,HTTPS连接同样支持复用,包括TLS会话复用。
## 调试技巧
```go
// 启用详细的HTTP跟踪
import "net/http/httptrace"
func debugWithTrace() {
trace := &httptrace.ClientTrace{
GotConn: func(info httptrace.GotConnInfo) {
fmt.Printf("连接复用: %v, 来自空闲池: %v\n",
info.Reused, info.WasIdle)
},
ConnectStart: func(network, addr string) {
fmt.Printf("开始连接: %s %s\n", network, addr)
},
ConnectDone: func(network, addr string, err error) {
if err != nil {
fmt.Printf("连接完成: %v\n", err)
}
},
}
ctx := httptrace.WithClientTrace(context.Background(), trace)
client := NewCurlx()
client.Get(ctx, "https://httpbin.org/get")
}
```
## 生产环境建议
1. **预热连接**:应用启动时进行连接预热
2. **监控指标**:监控连接池使用率和错误率
3. **优雅关闭**:应用关闭时清理连接资源
4. **负载均衡**:考虑使用连接池配合负载均衡
```go
// 生产环境推荐配置
func productionConfig() *Curlx {
return NewCurlx(
WithMaxIdleConns(200),
WithMaxIdleConnsPerHost(20),
WithMaxConnsPerHost(100),
WithIdleConnTimeout(120*time.Second),
SetOptionTimeOut(30*time.Second),
)
}
```
+190
View File
@@ -0,0 +1,190 @@
package example
import (
"context"
"fmt"
"net/http"
"sync"
"time"
"github.com/yuninks/curlx"
)
// ConnectionPoolManager 连接池管理器
type ConnectionPoolManager struct {
client *curlx.Curlx
transport *http.Transport
}
// NewConnectionPoolManager 创建连接池管理器
func NewConnectionPoolManager(opts ...curlx.Option) *ConnectionPoolManager {
// 添加连接池优化配置
poolOpts := append(opts,
WithConnectionPoolSettings(
100, // MaxIdleConns
10, // MaxIdleConnsPerHost
50, // MaxConnsPerHost
90*time.Second, // IdleConnTimeout
),
)
client := curlx.NewCurlx(poolOpts...)
return &ConnectionPoolManager{
client: client,
// transport: client.transport,
}
}
// WithConnectionPoolSettings 连接池配置选项
func WithConnectionPoolSettings(
maxIdleConns int,
maxIdleConnsPerHost int,
maxConnsPerHost int,
idleConnTimeout time.Duration,
) curlx.Option {
return func(options *curlx.ClientOptions) {
options.MaxIdleConns = maxIdleConns
options.MaxIdleConnsPerHost = maxIdleConnsPerHost
options.MaxConnsPerHost = maxConnsPerHost
options.IdleConnTimeout = idleConnTimeout
}
}
// ConcurrentRequests 并发请求演示
func (cpm *ConnectionPoolManager) ConcurrentRequests(urls []string) {
var wg sync.WaitGroup
results := make(chan string, len(urls))
startTime := time.Now()
// 并发执行多个请求
for i, url := range urls {
wg.Add(1)
go func(index int, targetURL string) {
defer wg.Done()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
start := time.Now()
response, err := cpm.client.Get(ctx, targetURL)
duration := time.Since(start)
if err != nil {
results <- fmt.Sprintf("Request %d to %s failed: %v (took %v)",
index, targetURL, err, duration)
} else {
results <- fmt.Sprintf("Request %d to %s succeeded: %d bytes (took %v)",
index, targetURL, len(response), duration)
}
}(i, url)
}
// 等待所有请求完成
wg.Wait()
close(results)
totalDuration := time.Since(startTime)
fmt.Printf("=== 并发请求完成 ===\n")
fmt.Printf("总耗时: %v\n", totalDuration)
fmt.Printf("平均每个请求: %v\n", totalDuration/time.Duration(len(urls)))
// 输出结果
for result := range results {
fmt.Println(result)
}
// 输出连接池状态
cpm.PrintPoolStats()
}
// PrintPoolStats 打印连接池统计信息
func (cpm *ConnectionPoolManager) PrintPoolStats() {
// stats := cpm.transport
fmt.Printf("\n=== 连接池统计 ===\n")
// fmt.Printf("当前空闲连接数: %d\n", stats.IdleConnCount())
// fmt.Printf("总连接数: %d\n", stats.TotalConnCount())
// fmt.Printf("等待队列长度: %d\n", stats.WaitQueueLength())
}
// ReuseExample 连接复用示例
func (cpm *ConnectionPoolManager) ReuseExample(baseURL string, requestCount int) {
fmt.Printf("=== 连接复用测试 ===\n")
fmt.Printf("目标URL: %s\n", baseURL)
fmt.Printf("请求次数: %d\n\n", requestCount)
startTime := time.Now()
for i := 0; i < requestCount; i++ {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
start := time.Now()
response, err := cpm.client.Get(ctx, baseURL)
duration := time.Since(start)
cancel() // 释放context资源
if err != nil {
fmt.Printf("第%d次请求失败: %v (耗时: %v)\n", i+1, err, duration)
} else {
fmt.Printf("第%d次请求成功: %d字节 (耗时: %v)\n", i+1, len(response), duration)
}
// 小间隔避免请求过于密集
time.Sleep(100 * time.Millisecond)
}
totalDuration := time.Since(startTime)
fmt.Printf("\n=== 测试完成 ===\n")
fmt.Printf("总耗时: %v\n", totalDuration)
fmt.Printf("平均每请求: %v\n", totalDuration/time.Duration(requestCount))
cpm.PrintPoolStats()
}
// PersistentConnectionExample 持久连接示例
func (cpm *ConnectionPoolManager) PersistentConnectionExample(targetURL string) {
fmt.Printf("=== 持久连接测试 ===\n")
fmt.Printf("测试URL: %s\n\n", targetURL)
// 预热连接
fmt.Println("预热阶段 - 建立初始连接...")
ctx1, cancel1 := context.WithTimeout(context.Background(), 10*time.Second)
_, err := cpm.client.Get(ctx1, targetURL)
cancel1()
if err != nil {
fmt.Printf("预热失败: %v\n", err)
return
}
cpm.PrintPoolStats()
// 实际测试
fmt.Println("\n实际测试阶段...")
testStart := time.Now()
for i := 1; i <= 5; i++ {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
start := time.Now()
response, err := cpm.client.Get(ctx, targetURL)
duration := time.Since(start)
cancel()
if err != nil {
fmt.Printf("第%d次请求: 失败 (%v) - 耗时: %v\n", i, err, duration)
} else {
fmt.Printf("第%d次请求: 成功 (%d字节) - 耗时: %v\n", i, len(response), duration)
}
time.Sleep(500 * time.Millisecond)
}
totalTime := time.Since(testStart)
fmt.Printf("\n持久连接测试完成,总耗时: %v\n", totalTime)
cpm.PrintPoolStats()
}
+149
View File
@@ -0,0 +1,149 @@
package example
import (
"context"
"fmt"
"testing"
"time"
"github.com/yuninks/curlx"
)
func TestConnectionReuse(t *testing.T) {
// 创建带优化连接池配置的客户端
client := curlx.NewCurlx(
curlx.WithMaxIdleConns(50),
curlx.WithMaxIdleConnsPerHost(10),
curlx.WithMaxConnsPerHost(20),
curlx.WithIdleConnTimeout(60*time.Second),
curlx.WithOptionTimeOut(30*time.Second),
)
// 测试同一个主机的多次请求,观察连接复用效果
targetURL := "https://httpbin.org/get"
fmt.Println("=== 连接复用测试开始 ===")
// 预热连接
fmt.Println("1. 预热连接...")
ctx1, cancel1 := context.WithTimeout(context.Background(), 10*time.Second)
_, err := client.Get(ctx1, targetURL)
cancel1()
if err != nil {
t.Logf("预热请求失败: %v", err)
return
}
fmt.Println(" 预热完成")
// 连续请求测试
fmt.Println("\n2. 连续请求测试...")
for i := 1; i <= 5; i++ {
start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
response, err := client.Get(ctx, targetURL)
duration := time.Since(start)
cancel()
if err != nil {
t.Logf("第%d次请求失败: %v (耗时: %v)", i, err, duration)
} else {
t.Logf("第%d次请求成功: %d字节 (耗时: %v)", i, len(response), duration)
}
time.Sleep(200 * time.Millisecond) // 短暂间隔
}
fmt.Println("\n=== 测试完成 ===")
}
func TestConcurrentConnectionReuse(t *testing.T) {
// 创建连接池管理器
poolManager := NewConnectionPoolManager(
curlx.WithMaxIdleConns(100),
curlx.WithMaxIdleConnsPerHost(20),
curlx.WithMaxConnsPerHost(30),
curlx.WithIdleConnTimeout(120*time.Second),
)
// 测试并发请求
urls := []string{
"https://httpbin.org/get",
"https://httpbin.org/uuid",
"https://httpbin.org/user-agent",
"https://httpbin.org/headers",
"https://httpbin.org/ip",
}
fmt.Println("=== 并发连接复用测试 ===")
poolManager.ConcurrentRequests(urls)
}
func TestPersistentConnection(t *testing.T) {
// 创建优化的客户端
client := curlx.NewCurlx(
curlx.WithMaxIdleConns(30),
curlx.WithMaxIdleConnsPerHost(5),
curlx.WithMaxConnsPerHost(15),
curlx.WithIdleConnTimeout(30*time.Second),
)
manager := &ConnectionPoolManager{
client: client,
// transport: client.transport,
}
fmt.Println("=== 持久连接测试 ===")
manager.PersistentConnectionExample("https://httpbin.org/delay/1")
}
func Example_connectionReuse() {
// 最佳实践示例:如何正确配置连接复用
// 1. 创建优化配置的客户端
client := curlx.NewCurlx(
// 连接池配置
curlx.WithMaxIdleConns(100), // 总空闲连接数
curlx.WithMaxIdleConnsPerHost(10), // 每主机空闲连接数
curlx.WithMaxConnsPerHost(50), // 每主机最大连接数
curlx.WithIdleConnTimeout(90*time.Second), // 空闲超时时间
// 其他优化配置
curlx.WithOptionTimeOut(30*time.Second),
)
// 2. 复用同一个客户端实例进行多次请求
ctx := context.Background()
// 第一次请求会建立新连接
response1, err := client.Get(ctx, "https://httpbin.org/get")
if err != nil {
fmt.Printf("首次请求失败: %v\n", err)
return
}
fmt.Printf("首次请求成功: %d字节\n", len(response1))
// 后续请求会复用已有连接
response2, err := client.Get(ctx, "https://httpbin.org/uuid")
if err != nil {
fmt.Printf("第二次请求失败: %v\n", err)
return
}
fmt.Printf("第二次请求成功: %d字节\n", len(response2))
// 3. 查看连接池状态
manager := &ConnectionPoolManager{
client: client,
// transport: client.transport,
}
manager.PrintPoolStats()
// Output:
// 首次请求成功: [字节数]
// 第二次请求成功: [字节数]
// === 连接池统计 ===
// 当前空闲连接数: 1
// 总连接数: 1
// 等待队列长度: 0
}
+1
View File
@@ -3,6 +3,7 @@ module github.com/yuninks/curlx
go 1.20
require (
code.yun.ink/pkg/convx v1.0.3
github.com/tidwall/gjson v1.17.0
golang.org/x/net v0.18.0
)
+2
View File
@@ -1,3 +1,5 @@
code.yun.ink/pkg/convx v1.0.3 h1:pH8dUOgsoaBYVQ3+4C2+uVua561nDxq6/GpaQ9wnCew=
code.yun.ink/pkg/convx v1.0.3/go.mod h1:6xqmUend1kwarRvJ0TQlfzzS4QCWdRrXQiUY/ggzYqo=
github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+69 -15
View File
@@ -6,35 +6,55 @@ import (
"time"
)
type clientOptions struct {
type ClientOptions struct {
TimeOut time.Duration
InsecureSkipVerify bool
Logger OptionLogger
LoggerLength int // 日志输出长度
CertFingerprint string // 证书指纹验证
// 连接池配置
MaxIdleConns int
MaxIdleConnsPerHost int
MaxConnsPerHost int
IdleConnTimeout time.Duration
}
func defaultOptions() clientOptions {
return clientOptions{
TimeOut: time.Second * 120, // 默认超时120
func defaultOptions() ClientOptions {
return ClientOptions{
TimeOut: time.Second * 120, // 默认超时120
Logger: defaultLogger{},
LoggerLength: 100,
MaxIdleConns: 100, // 默认连接池大小
MaxIdleConnsPerHost: 10, // 每主机默认空闲连接数
MaxConnsPerHost: 50, // 每主机最大连接数
IdleConnTimeout: 90 * time.Second, // 空闲连接超时
}
}
type Option func(*clientOptions)
type Option func(*ClientOptions)
/**
* 设置超时时间
*/
func SetOptionTimeOut(t time.Duration) Option {
return func(options *clientOptions) {
func WithOptionTimeOut(t time.Duration) Option {
return func(options *ClientOptions) {
options.TimeOut = t
}
}
// 添加证书指纹验证选项
func WithOptionTLSPin(certFingerprint string) Option {
return func(options *ClientOptions) {
options.CertFingerprint = certFingerprint
}
}
/**
* 不校验HTTPS证书
*/
func SetOptionTLSInsecureSkipVerify() Option {
return func(options *clientOptions) {
func WithOptionTLSInsecureSkipVerify() Option {
return func(options *ClientOptions) {
options.InsecureSkipVerify = true
}
}
@@ -42,25 +62,59 @@ func SetOptionTLSInsecureSkipVerify() Option {
/**
* 设置日志输出
*/
func SetOptionLog(log OptionLogger) Option {
return func(options *clientOptions) {
func WithOptionLog(log OptionLogger) Option {
return func(options *ClientOptions) {
options.Logger = log
}
}
/**
* 设置日志输出长度
*/
func WithOptionLoggerLength(length int) Option {
return func(options *ClientOptions) {
options.LoggerLength = length
}
}
// 连接池配置选项
func WithMaxIdleConns(maxIdleConns int) Option {
return func(options *ClientOptions) {
options.MaxIdleConns = maxIdleConns
}
}
func WithMaxIdleConnsPerHost(maxIdleConnsPerHost int) Option {
return func(options *ClientOptions) {
options.MaxIdleConnsPerHost = maxIdleConnsPerHost
}
}
func WithMaxConnsPerHost(maxConnsPerHost int) Option {
return func(options *ClientOptions) {
options.MaxConnsPerHost = maxConnsPerHost
}
}
func WithIdleConnTimeout(timeout time.Duration) Option {
return func(options *ClientOptions) {
options.IdleConnTimeout = timeout
}
}
type OptionLogger interface {
Infof(ctx context.Context, format string, args ...interface{})
Errorf(ctx context.Context, format string, args ...interface{})
Infof(ctx context.Context, format string, args ...any)
Errorf(ctx context.Context, format string, args ...any)
}
type defaultLogger struct{}
func (defaultLogger) Errorf(ctx context.Context, format string, args ...interface{}) {
func (defaultLogger) Errorf(ctx context.Context, format string, args ...any) {
// 输出日志
log.Printf(format, args...)
}
func (defaultLogger) Infof(ctx context.Context, format string, args ...interface{}) {
func (defaultLogger) Infof(ctx context.Context, format string, args ...any) {
// 输出日志
log.Printf(format, args...)
}
+86 -17
View File
@@ -1,17 +1,22 @@
package curlx
import (
"encoding/json"
"net/http"
)
type ClientParams struct {
Url string
Method Method // GET/POST
Body interface{}
Headers map[string]interface{}
Cookies interface{}
Method Method // GET/POST/PUT/DELETE
Body []byte
Headers http.Header
Cookies []http.Cookie
ContentType ContentType // FORM,JSON,XML
}
func defaultParams() ClientParams {
return ClientParams{
Headers: map[string]interface{}{}, // 初始化map
Headers: http.Header{},
}
}
@@ -49,9 +54,22 @@ func SetParamsMethod(m Method) Param {
/**
* 设置参数
*/
func SetParamsBody(p interface{}) Param {
func SetParamsBody(by []byte) Param {
return func(param *ClientParams) {
param.Body = p
param.Body = by
}
}
func SetParamsBodyAny(v interface{}) Param {
return func(param *ClientParams) {
switch value := v.(type) {
case []byte:
param.Body = value
case string:
param.Body = []byte(value)
default:
param.Body, _ = json.Marshal(value)
}
}
}
@@ -60,12 +78,18 @@ func SetParamsBody(p interface{}) Param {
*/
func SetParamsFormText(fieldName, fieldValue string) Param {
return func(param *ClientParams) {
fp := param.Body.([]FormParam)
fp = append(fp, FormParam{
m := []FormParam{}
if param.Body != nil {
json.Unmarshal(param.Body, &m)
}
m = append(m, FormParam{
FieldName: fieldName,
FieldValue: fieldValue,
FieldType: FieldTypeText,
})
fp, _ := json.Marshal(m)
param.Body = fp
}
}
@@ -75,25 +99,49 @@ func SetParamsFormText(fieldName, fieldValue string) Param {
*/
func SetParamsFormFile(fieldName, fileName string, fileBytes []byte) Param {
return func(param *ClientParams) {
fp := param.Body.([]FormParam)
fp := []FormParam{}
if param.Body != nil {
json.Unmarshal(param.Body, &fp)
}
fp = append(fp, FormParam{
FieldName: fieldName,
FieldType: FieldTypeFile,
FileName: fileName,
FileBytes: fileBytes,
})
param.Body = fp
fpb, _ := json.Marshal(fp)
param.Body = fpb
}
}
/**
* 设置请求头
*/
func SetParamsHeaders(h map[string]interface{}) Param {
func SetParamsHeaders(h map[string]string) Param {
return func(param *ClientParams) {
for key, _ := range h {
param.Headers[key] = h[key]
if param.Headers == nil {
param.Headers = http.Header{}
}
for k, v := range h {
param.Headers.Set(k, v)
}
}
}
/**
* 设置请求头
*/
func SetParamsHeader(key, value string) Param {
return func(param *ClientParams) {
if param.Headers == nil {
param.Headers = http.Header{}
}
param.Headers.Add(key, value)
}
}
@@ -102,7 +150,25 @@ func SetParamsHeaders(h map[string]interface{}) Param {
*/
func SetUserAgent(userAgent UserAgent) Param {
return func(param *ClientParams) {
param.Headers["User-Agent"] = string(userAgent)
if param.Headers == nil {
param.Headers = http.Header{}
}
param.Headers.Set("User-Agent", string(userAgent))
}
}
func SetCookie(name, value string) Param {
return func(param *ClientParams) {
param.Cookies = append(param.Cookies, http.Cookie{
Name: name,
Value: value,
})
}
}
func SetCookies(cookies []http.Cookie) Param {
return func(param *ClientParams) {
param.Cookies = append(param.Cookies, cookies...)
}
}
@@ -111,14 +177,17 @@ func SetUserAgent(userAgent UserAgent) Param {
*/
func SetReferer(referer string) Param {
return func(param *ClientParams) {
param.Headers["Referer"] = referer
if param.Headers == nil {
param.Headers = http.Header{}
}
param.Headers.Set("Referer", referer)
}
}
/**
* 设置cookies
*/
func SetParamsCookies(c interface{}) Param {
func SetParamsCookies(c []http.Cookie) Param {
return func(param *ClientParams) {
param.Cookies = c
}
+60 -169
View File
@@ -3,14 +3,14 @@ package curlx
import (
"bytes"
"encoding/json"
"encoding/xml"
"errors"
"io"
"mime/multipart"
"net/http"
"net/url"
"strconv"
"strings"
"code.yun.ink/pkg/convx"
)
/**
@@ -38,25 +38,13 @@ func (p *ClientParams) parseUrl() error {
* 处理请求头Header
*/
func (p *ClientParams) parseHeaders(r *http.Request) {
if p.Headers != nil {
if r.Header.Get("User-Agent") == "" {
r.Header.Add("User-Agent", string(UserAgentChrome))
}
for k, v := range p.Headers {
switch value := v.(type) {
case string:
r.Header.Set(k, value)
case []string:
for _, vv := range value {
r.Header.Add(k, vv)
}
case ContentType:
r.Header.Set(k, string(value))
case UserAgent:
r.Header.Set(k, string(value))
}
}
if p.Headers.Get("User-Agent") == "" {
p.Headers.Add("User-Agent", string(UserAgentChrome))
}
r.Header = p.Headers
}
/**
@@ -67,35 +55,29 @@ func (p *ClientParams) parseParams() (str io.Reader, err error) {
// 初始化(如未初始化)
if p.Headers == nil {
p.Headers = make(map[string]interface{})
p.Headers = http.Header{}
}
if p.Body != nil {
if p.ContentType == ContentTypeJson {
// 判断是否存在
// 添加Content-Type
if _, ok := p.Headers["Content-Type"]; !ok {
p.Headers["Content-Type"] = ContentTypeJson
}
strParam, ok := p.Body.(string)
if ok {
return bytes.NewReader([]byte(strParam)), nil
}
b, err := json.Marshal(p.Body)
if err == nil {
return bytes.NewReader(b), nil
}
} else if p.ContentType == ContentTypeForm {
// 表单上传(可能有文件)
// 文件上传的
params := []FormParam{}
if value, ok := p.Body.([]FormParam); ok {
params = value
} else if value, ok := p.Body.(FormParam); ok {
params = append(params, value)
} else {
return nil, errors.New("表单上传的参数格式不正确")
p.Headers.Set("Content-Type", string(p.ContentType))
}
if len(p.Body) == 0 {
return nil, nil
}
switch p.ContentType {
case ContentTypeJson:
// JSON
return bytes.NewReader(p.Body), nil
case ContentTypeForm:
// 表单
params := []FormParam{}
err = json.Unmarshal(p.Body, &params)
if err != nil {
return nil, err
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
for _, v := range params {
@@ -107,126 +89,50 @@ func (p *ClientParams) parseParams() (str io.Reader, err error) {
}
}
writer.Close()
p.Headers["Content-Type"] = writer.FormDataContentType()
p.Headers.Set("Content-Type", writer.FormDataContentType())
return body, nil
} else if p.ContentType == ContentTypeXml {
if _, ok := p.Headers["Content-Type"]; !ok {
p.Headers["Content-Type"] = ContentTypeXml
case ContentTypeXml:
// XML
return bytes.NewReader(p.Body), nil
case ContentTypeText:
// TEXT
return bytes.NewReader(p.Body), nil
case ContentTypeUrlEncoded:
// URL编码
m := map[string]any{}
if err = json.Unmarshal(p.Body, &m); err != nil {
return nil, err
}
var string_data string
if value, ok := p.Body.(string); ok {
string_data = string(value)
} else {
var by []byte
by, err = xml.Marshal(p.Body)
if err != nil {
return
}
string_data = string(by)
}
return strings.NewReader(string_data), nil
} else if p.ContentType == ContentTypeText {
if _, ok := p.Headers["Content-Type"]; !ok {
p.Headers["Content-Type"] = ContentTypeText
}
var string_data string
if value, ok := p.Body.(string); ok {
string_data = string(value)
} else {
err = errors.New("TEXT类型的参数仅支持字符串")
return
}
return strings.NewReader(string_data), nil
} else if p.ContentType == ContentTypeUrlEncoded {
if _, ok := p.Headers["Content-Type"]; !ok {
p.Headers["Content-Type"] = "application/x-www-form-urlencoded"
}
// 判断需要map[string]interface{}类型
paramValue, ok := p.Body.(map[string]interface{})
if !ok {
return strings.NewReader(""), errors.New("参数需map[string]interface{}")
}
values := url.Values{}
for k, v := range paramValue {
// 字符串
if v_string, ok := v.(string); ok {
values.Set(k, v_string)
}
// 字符串切片
if vv, ok := v.([]string); ok {
for _, vvv := range vv {
values.Add(k+"[]", vvv)
}
}
// int转string
if v_int, ok := v.(int); ok {
values.Set(k, strconv.Itoa(v_int))
}
// int64转string
if v_int64, ok := v.(int64); ok {
values.Set(k, strconv.FormatInt(v_int64, 10))
}
// float32转string
if v_float32, ok := v.(float32); ok {
values.Set(k, strconv.FormatFloat(float64(v_float32), 'f', -1, 32))
}
// float64转string
if v_float64, ok := v.(float64); ok {
values.Set(k, strconv.FormatFloat(v_float64, 'f', -1, 64))
}
for k, v := range m {
val := convx.ToString(v)
values.Set(k, val)
}
return strings.NewReader(values.Encode()), nil
} else {
// 如果是GET请求
default:
if p.Method == MethodGet {
// 判断需要map[string]interface{}类型
paramValue, ok := p.Body.(map[string]interface{})
if !ok {
return strings.NewReader(""), errors.New("参数需map[string]interface{}")
m := map[string]any{}
if err = json.Unmarshal(p.Body, &m); err != nil {
return nil, err
}
// 拼接参数到URL
if strings.Contains(p.Url, "?") {
p.Url += "&"
} else {
p.Url += "?"
}
for k, v := range paramValue {
// 字符串
if v_string, ok := v.(string); ok {
p.Url += k + "=" + v_string + "&"
}
// 字符串切片
if vv, ok := v.([]string); ok {
for _, vvv := range vv {
p.Url += k + "[]=" + vvv + "&"
}
}
// int转string
if v_int, ok := v.(int); ok {
p.Url += k + "=" + strconv.Itoa(v_int) + "&"
}
// int64转string
if v_int64, ok := v.(int64); ok {
p.Url += k + "=" + strconv.FormatInt(v_int64, 10) + "&"
}
// float32转string
if v_float32, ok := v.(float32); ok {
p.Url += k + "=" + strconv.FormatFloat(float64(v_float32), 'f', -1, 32) + "&"
}
// float64转string
if v_float64, ok := v.(float64); ok {
p.Url += k + "=" + strconv.FormatFloat(v_float64, 'f', -1, 64) + "&"
url, err := url.Parse(p.Url) // 解析URL
if err != nil {
return nil, err
}
query := url.Query()
for k, v := range m {
val := convx.ToString(v)
query[k] = append(query[k], val)
}
url.RawQuery = query.Encode()
p.Url = url.String()
} else {
return nil, errors.New("curlx 不支持的数据类型")
}
}
}
return
@@ -236,23 +142,8 @@ func (p *ClientParams) parseParams() (str io.Reader, err error) {
* 处理Cookie
*/
func (p *ClientParams) parseCookies(r *http.Request) {
switch p.Cookies.(type) {
case string:
cookies := p.Cookies.(string)
r.Header.Add("Cookie", cookies)
case map[string]string:
cookies := p.Cookies.(map[string]string)
for k, v := range cookies {
r.AddCookie(&http.Cookie{
Name: k,
Value: v,
})
}
case []*http.Cookie:
cookies := p.Cookies.([]*http.Cookie)
for _, cookie := range cookies {
r.AddCookie(cookie)
}
for _, cookie := range p.Cookies {
r.AddCookie(&cookie)
}
}
+70 -43
View File
@@ -1,6 +1,8 @@
package curlx
import (
"compress/gzip"
"io"
"net"
"net/http"
"strings"
@@ -10,62 +12,73 @@ import (
// Response response object
type Response struct {
resp *http.Response
req *http.Request
response *http.Response
request *http.Request
body []byte
err error
}
// ResponseBody response body
type ResponseBody []byte
// String fmt outout
func (r ResponseBody) String() string {
return string(r)
func (l *Response) Close() error {
if l.response.Body != nil {
return l.response.Body.Close()
}
// Read get slice of response body
func (r ResponseBody) Read(length int) []byte {
if length > len(r) {
length = len(r)
}
return r[:length]
}
// GetContents format response body as string
func (r ResponseBody) GetContents() string {
return string(r)
return nil
}
// GetRequest get request object
func (r *Response) GetRequest() *http.Request {
return r.req
return r.request
}
func (r *Response) GetResponse() *http.Response {
return r.response
}
func (r *Response) GetError() error {
return r.err
}
// GetBody parse response body
func (r *Response) GetBody() (ResponseBody, error) {
return ResponseBody(r.body), r.err
func (r *Response) GetBody() ([]byte, error) {
if r.err != nil {
return nil, r.err
}
if r.body != nil {
return r.body, nil
}
if r.response == nil {
return nil, nil
}
body := []byte{}
var err error
if r.response.Header.Get("Content-Encoding") == "gzip" {
reader, err := gzip.NewReader(r.response.Body)
if err != nil {
return nil, err
}
defer reader.Close()
body, err = io.ReadAll(reader)
if err != nil {
return nil, err
}
} else {
body, err = io.ReadAll(r.response.Body)
if err != nil {
return nil, err
}
}
// close body
r.response.Body.Close()
r.body = body
return body, err
}
// GetParsedBody parse response body with gjson
func (r *Response) GetParsedBody() (*gjson.Result, error) {
pb := gjson.ParseBytes(r.body)
return &pb, nil
func (r Response) GetStatusCode() int {
if r.response == nil {
return 0
}
// GetStatusCode get response status code
func (r *Response) GetStatusCode() int {
return r.resp.StatusCode
}
// GetReasonPhrase get response reason phrase
func (r *Response) GetReasonPhrase() string {
status := r.resp.Status
arr := strings.Split(status, " ")
return arr[1]
return r.response.StatusCode
}
// IsTimeout get if request is timeout
@@ -84,16 +97,30 @@ func (r *Response) IsTimeout() bool {
return false
}
// GetParsedBody parse response body with gjson
func (r *Response) GetParsedBody() (*gjson.Result, error) {
body, err := r.GetBody()
if err != nil {
return nil, err
}
pb := gjson.ParseBytes(body)
return &pb, nil
}
// GetHeaders get response headers
func (r *Response) GetHeaders() map[string][]string {
return r.resp.Header
if r.response == nil {
return nil
}
return r.response.Header
}
// GetHeader get response header
func (r *Response) GetHeader(name string) []string {
headers := r.GetHeaders()
for k, v := range headers {
if strings.ToLower(name) == strings.ToLower(k) {
if strings.EqualFold(name, k) {
return v
}
}
-12
View File
@@ -1,12 +0,0 @@
package curlx
type UserAgent string
const(
UserAgentChrome UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
UserAgentFirefox UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 "
UserAgentIE UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; "
UserAgentEdge UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
UserAgentWechat UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
)