Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f15c882874 | ||
|
|
3e96b1ada6 | ||
|
|
d6e1229961 | ||
|
|
3f51fb39cb | ||
|
|
0427f4a844 | ||
|
|
660f3dee83 | ||
|
|
0931539e1c | ||
|
|
8bca4cc013 | ||
|
|
6a9f388bb6 | ||
|
|
14bb42c131 | ||
|
|
cd521e4837 | ||
|
|
aa0e094851 | ||
|
|
566d0b7b51 | ||
|
|
6387f372c4 | ||
|
|
2cf94d88ab |
@@ -0,0 +1,17 @@
|
||||
package curlx
|
||||
|
||||
import "errors"
|
||||
|
||||
type UserAgent string
|
||||
|
||||
const (
|
||||
UserAgentChrome UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
UserAgentFirefox UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 "
|
||||
UserAgentIE UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; "
|
||||
UserAgentEdge UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
UserAgentWechat UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
|
||||
)
|
||||
|
||||
var (
|
||||
ErrStatusNotOK error = errors.New("Status not ok")
|
||||
)
|
||||
@@ -2,10 +2,8 @@ package curlx
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -22,7 +20,7 @@ import (
|
||||
// type DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
|
||||
type Curlx struct {
|
||||
opts clientOptions
|
||||
opts ClientOptions
|
||||
transport *http.Transport
|
||||
}
|
||||
|
||||
@@ -33,31 +31,13 @@ func NewCurlx(opts ...Option) *Curlx {
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
// Dial: func(netw, addr string) (net.Conn, error) {
|
||||
// // 这里指定域名访问的IP
|
||||
// // if addr == "api.hk.blueoceanpay.com:443" {
|
||||
// // addr = "47.56.200.21:443"
|
||||
// // }
|
||||
// conn, err := net.DialTimeout(netw, addr, time.Second*time.Duration(timeOut)) // 设置建立连接超时
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// // conn.RemoteAddr().String()
|
||||
// conn.SetDeadline(time.Now().Add(time.Second * time.Duration(timeOut))) // 设置发送接收数据超时
|
||||
// return conn, nil
|
||||
// },
|
||||
// DialContext: (&net.Dialer{
|
||||
// Timeout: 3 * time.Second, // 建立TCP链接的超时时间
|
||||
// KeepAlive: 30 * time.Second, // TCP keepalive超时时间
|
||||
// }).DialContext,
|
||||
// TLSHandshakeTimeout: time.Second * 10, // TLS握手超时
|
||||
// ResponseHeaderTimeout: time.Second * 10, // 接收响应头的超时时间
|
||||
// ExpectContinueTimeout: time.Second * 10, // 发送请求头超时时间 100-continue状态码超时时间
|
||||
DisableKeepAlives: false, // 短连接(默认是使用长连接,连接过多时会造成服务器拒绝服务问题)
|
||||
MaxIdleConns: 0, // 所有host的连接池最大连接数量,默认无穷大
|
||||
MaxIdleConnsPerHost: 5, // 每个host的连接池最大空闲连接收,默认2
|
||||
MaxConnsPerHost: 0, // 每个host的最大连接数量
|
||||
IdleConnTimeout: time.Second * 2, // 空闲连接超时关闭的时间
|
||||
DisableKeepAlives: false, // 启用keep-alive连接复用
|
||||
MaxIdleConns: defaultOpts.MaxIdleConns,
|
||||
MaxIdleConnsPerHost: defaultOpts.MaxIdleConnsPerHost,
|
||||
MaxConnsPerHost: defaultOpts.MaxConnsPerHost,
|
||||
IdleConnTimeout: defaultOpts.IdleConnTimeout,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
if defaultOpts.InsecureSkipVerify {
|
||||
@@ -79,10 +59,14 @@ func (c *Curlx) WithProxySocks5(address string) error {
|
||||
baseDialer := &net.Dialer{
|
||||
// Timeout: 180 * time.Second,
|
||||
// KeepAlive: 180 * time.Second,
|
||||
Resolver: &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: c.transport.DialContext,
|
||||
},
|
||||
}
|
||||
dialSocksProxy, err := proxy.SOCKS5("tcp", address, nil, baseDialer)
|
||||
if err != nil {
|
||||
fmt.Println("proxy.SOCKS5 err", err)
|
||||
c.opts.Logger.Errorf(context.Background(), "proxy.SOCKS5 err: %v", err)
|
||||
return err
|
||||
}
|
||||
dialContext := (baseDialer).DialContext
|
||||
@@ -94,12 +78,13 @@ func (c *Curlx) WithProxySocks5(address string) error {
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用HTTP代理
|
||||
* 使用HTTP/HTTPS代理
|
||||
* @param proxyAddr "https://proxyserver:port"
|
||||
*/
|
||||
func (c *Curlx) WithProxyHttp(proxyAddr string) error {
|
||||
proxy, err := url.Parse(proxyAddr)
|
||||
if err != nil {
|
||||
c.opts.Logger.Errorf(context.Background(), "proxy.HTTP/HTTPS err: %v", err)
|
||||
return err
|
||||
}
|
||||
c.transport.Proxy = http.ProxyURL(proxy)
|
||||
@@ -118,154 +103,110 @@ func (c *Curlx) WithAddress(ctx context.Context, addr string) {
|
||||
/**
|
||||
* 简单请求
|
||||
*/
|
||||
func (c *Curlx) Send(ctx context.Context, p ...Param) (res []byte, httpcode int, err error) {
|
||||
_, response, err := c.SendExec(ctx, p...)
|
||||
func (c *Curlx) Send(ctx context.Context, p ...Param) (res []byte, err error) {
|
||||
resp := c.exec(ctx, p...)
|
||||
if resp.Err != nil {
|
||||
return nil, resp.Err
|
||||
}
|
||||
defer resp.Close() // 处理完关闭
|
||||
|
||||
status := resp.GetStatusCode()
|
||||
if status != 200 {
|
||||
c.opts.Logger.Errorf(ctx, "curlx.Send status not OK: %d", status)
|
||||
return nil, ErrStatusNotOK
|
||||
}
|
||||
|
||||
body, err := resp.GetBody()
|
||||
if err != nil {
|
||||
return nil, -1, err
|
||||
c.opts.Logger.Errorf(ctx, "curlx.Send getBody err:%v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer response.Body.Close() // 处理完关闭
|
||||
|
||||
// stdout := os.Stdout // 将结果定位到标准输出,也可以直接打印出来,或定位到其他地方进行相应处理
|
||||
// _, err = io.Copy(stdout, response.Body) // 将第二个参数拷贝到第一个参数,直到第二参数到达EOF或发生错误,返回拷贝的值
|
||||
status := response.StatusCode // 获取状态码,正常是200
|
||||
|
||||
var body []byte
|
||||
// switch response.Header.Get("Content-Encoding") {
|
||||
// case "gzip":
|
||||
// reader, err := gzip.NewReader(response.Body)
|
||||
// if err != nil {
|
||||
// return nil, status, err
|
||||
// }
|
||||
// for {
|
||||
// buf := make([]byte, 1024)
|
||||
// n, err := reader.Read(buf)
|
||||
// if err != nil && err != io.EOF {
|
||||
// panic(err)
|
||||
// }
|
||||
// if n == 0 {
|
||||
// break
|
||||
// }
|
||||
// body = append(body, buf...)
|
||||
// }
|
||||
// default:
|
||||
// body, _ = io.ReadAll(response.Body)
|
||||
// }
|
||||
|
||||
if response.Header.Get("Content-Encoding") == "gzip" {
|
||||
reader, err := gzip.NewReader(response.Body)
|
||||
if err != nil {
|
||||
return nil, response.StatusCode, err
|
||||
}
|
||||
body, err = io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, response.StatusCode, err
|
||||
}
|
||||
defer reader.Close()
|
||||
} else {
|
||||
body, err = io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, response.StatusCode, err
|
||||
}
|
||||
// 打印日志时截取前指定长度,避免日志过大
|
||||
bodyLog := []rune(string(body))
|
||||
if len(bodyLog) > c.opts.LoggerLength {
|
||||
bodyLog = bodyLog[:c.opts.LoggerLength]
|
||||
}
|
||||
|
||||
c.opts.Logger.Infof(ctx, "curlx.Send body:%s", string(body))
|
||||
return body, status, nil
|
||||
c.opts.Logger.Infof(ctx, "curlx.Send response body:%s", string(bodyLog))
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c *Curlx) SendWithResponee(ctx context.Context, ps ...Param) Response {
|
||||
r := Response{}
|
||||
req, resp, err := c.SendExec(ctx, ps...)
|
||||
r.req = req
|
||||
r.resp = resp
|
||||
// PostJson 发送JSON数据
|
||||
func (l *Curlx) PostJson(ctx context.Context, url string, jsonStr string) ([]byte, error) {
|
||||
return l.Send(ctx,
|
||||
SetParamsUrl(url),
|
||||
SetParamsBody([]byte(jsonStr)),
|
||||
SetParamsContentType(ContentTypeJson),
|
||||
SetParamsMethod(MethodPost),
|
||||
)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return r
|
||||
}
|
||||
var body []byte
|
||||
// switch resp.Header.Get("Content-Encoding") {
|
||||
// case "gzip":
|
||||
// reader, err := gzip.NewReader(resp.Body)
|
||||
// if err != nil {
|
||||
// r.err = err
|
||||
// return r
|
||||
// }
|
||||
// for {
|
||||
// buf := make([]byte, 1024)
|
||||
// n, err := reader.Read(buf)
|
||||
// if err != nil && err != io.EOF {
|
||||
// panic(err)
|
||||
// }
|
||||
// if n == 0 {
|
||||
// break
|
||||
// }
|
||||
// // 读取n个字节
|
||||
// body = append(body, buf[:n]...)
|
||||
// // body = append(body, buf...)
|
||||
// }
|
||||
// default:
|
||||
// body, _ = io.ReadAll(resp.Body)
|
||||
// }
|
||||
// Get 简单GET请求
|
||||
func (l *Curlx) Get(ctx context.Context, url string) ([]byte, error) {
|
||||
return l.Send(ctx,
|
||||
SetParamsUrl(url),
|
||||
SetParamsMethod(MethodGet),
|
||||
)
|
||||
}
|
||||
|
||||
if resp.Header.Get("Content-Encoding") == "gzip" {
|
||||
reader, err := gzip.NewReader(resp.Body)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return r
|
||||
}
|
||||
body, err = io.ReadAll(reader)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return r
|
||||
}
|
||||
defer reader.Close()
|
||||
} else {
|
||||
body, err = io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
r.body = body
|
||||
c.opts.Logger.Infof(ctx, "curlx.Send body:%s", string(body))
|
||||
return r
|
||||
func (c *Curlx) SendWithResponse(ctx context.Context, ps ...Param) Response {
|
||||
return c.exec(ctx, ps...)
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行发送
|
||||
* 注意:外部使用需要加这一句 defer response.Body.Close()
|
||||
*/
|
||||
func (c *Curlx) SendExec(ctx context.Context, ps ...Param) (req *http.Request, resp *http.Response, err error) {
|
||||
func (c *Curlx) exec(ctx context.Context, ps ...Param) Response {
|
||||
resp := Response{}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: c.opts.TimeOut, // 整个请求的超时时间 设置该条连接的超时
|
||||
Transport: c.transport, //
|
||||
}
|
||||
|
||||
// 在http.Client中添加CheckRedirect函数 实现重定向控制
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 { // 限制重定向次数
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
p := defaultParams()
|
||||
for _, param := range ps {
|
||||
param(&p)
|
||||
}
|
||||
c.opts.Logger.Infof(ctx, "curlx.sendExec params:%+v", p)
|
||||
|
||||
err = p.parseMethod()
|
||||
// 截取Body前指定长度输出,避免日志过大
|
||||
bodyLog := []rune(string(p.Body))
|
||||
if len(bodyLog) > c.opts.LoggerLength {
|
||||
bodyLog = bodyLog[:c.opts.LoggerLength]
|
||||
}
|
||||
|
||||
c.opts.Logger.Infof(ctx, "curlx.sendExec params url:%s method:%s contentType:%s body:%s headers:%+v cookies:%+v", p.Url, p.Method, p.ContentType, string(bodyLog), p.Headers, p.Cookies)
|
||||
|
||||
err := p.parseMethod()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
c.opts.Logger.Errorf(ctx, "curlx.sendExec parseMethod err:%v", err)
|
||||
resp.Err = err
|
||||
return resp
|
||||
}
|
||||
|
||||
// 判断和处理url
|
||||
err = p.parseUrl()
|
||||
if err != nil {
|
||||
c.opts.Logger.Errorf(ctx, "curlx.sendExec parseUrl err:%v", err)
|
||||
return nil, nil, err
|
||||
resp.Err = err
|
||||
return resp
|
||||
}
|
||||
|
||||
// 处理参数
|
||||
reqParams, err := p.parseParams()
|
||||
if err != nil {
|
||||
c.opts.Logger.Errorf(ctx, "curlx.sendExec parseParams err:%v", err)
|
||||
return nil, nil, err
|
||||
resp.Err = err
|
||||
return resp
|
||||
}
|
||||
|
||||
// 初始化句柄
|
||||
@@ -276,9 +217,13 @@ func (c *Curlx) SendExec(ctx context.Context, ps ...Param) (req *http.Request, r
|
||||
)
|
||||
if err != nil {
|
||||
c.opts.Logger.Errorf(ctx, "curlx.sendExec NewRequest err:%v", err)
|
||||
return nil, nil, err
|
||||
resp.Err = err
|
||||
return resp
|
||||
}
|
||||
|
||||
c.opts.Logger.Infof(ctx, "curlx.sendExec request:%+v", request)
|
||||
resp.Request = request
|
||||
|
||||
// 这里指定要访问的HOST,到时候服务器获取主机是获取到这个
|
||||
// request.Host = "api.hk.blueoceantech.co"
|
||||
|
||||
@@ -295,10 +240,12 @@ func (c *Curlx) SendExec(ctx context.Context, ps ...Param) (req *http.Request, r
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
c.opts.Logger.Errorf(ctx, "curlx.sendExec client.Do err:%v", err)
|
||||
return nil, nil, err
|
||||
resp.Err = err
|
||||
return resp
|
||||
}
|
||||
// response.StatusCode
|
||||
return request, response, nil
|
||||
resp.Response = response
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -311,16 +258,15 @@ func (c *Curlx) SendStream(ctx context.Context, ps ...Param) (<-chan string, err
|
||||
go func() {
|
||||
defer close(data)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), c.opts.TimeOut)
|
||||
defer cancel()
|
||||
|
||||
_, response, err := c.SendExec(ctx, ps...)
|
||||
if err != nil {
|
||||
response := c.exec(ctx, ps...)
|
||||
if response.Err != nil {
|
||||
return
|
||||
}
|
||||
defer response.Body.Close() // 处理完关闭
|
||||
|
||||
scanner := bufio.NewScanner(response.Body)
|
||||
defer response.Close() // 处理完关闭
|
||||
scanner := bufio.NewScanner(response.Response.Body)
|
||||
for scanner.Scan() {
|
||||
text := scanner.Text()
|
||||
if text == "" {
|
||||
|
||||
+19
-7
@@ -2,18 +2,20 @@ package curlx
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGet(t *testing.T) {
|
||||
resp, code, err := NewCurlx().Send(context.Background(),
|
||||
resp, err := NewCurlx().Send(context.Background(),
|
||||
SetParamsUrl("https://www.baidu.com"),
|
||||
SetParamsMethod(MethodGet),
|
||||
)
|
||||
t.Log(resp, code, err)
|
||||
t.Log(string(resp), err)
|
||||
|
||||
}
|
||||
|
||||
@@ -36,15 +38,25 @@ func TestForm(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
by, _ := json.Marshal(s)
|
||||
|
||||
p := ClientParams{
|
||||
Url: "http://tech-dev.sealmoo.com/api/material/upload",
|
||||
Method: "POST",
|
||||
Body: s,
|
||||
Headers: map[string]interface{}{
|
||||
"Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0ZW5hbnRfaWQiOjAsImNsaWVudF9pZCI6MCwidXNlcl9pZCI6MSwiZXhwIjoxNzAxMzk3NzkxfQ.9_uJ6y8I4JZTwgSenwHC_01nddLuI4zmgpyPhn5M6j8",
|
||||
Body: by,
|
||||
Headers: http.Header{
|
||||
"Content-Type": []string{"application/json"},
|
||||
"Authorization": []string{"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0ZW5hbnRfaWQiOjAsImNsaWVudF9pZCI6MCwidXNlcl9pZCI6MSwiZXhwIjoxNzAxMzk3NzkxfQ.9_uJ6y8I4JZTwgSenwHC_01nddLuI4zmgpyPhn5M6j8"},
|
||||
},
|
||||
ContentType: ContentTypeForm,
|
||||
}
|
||||
resp, code, err := NewCurlx().Send(context.Background(), SetParamsAll(p))
|
||||
fmt.Println(resp, code, err)
|
||||
resp, err := NewCurlx().Send(context.Background(), SetParamsAll(p))
|
||||
fmt.Println(resp, err)
|
||||
}
|
||||
|
||||
func TestProxy(t *testing.T) {
|
||||
c := NewCurlx()
|
||||
c.WithProxySocks5("127.0.0.1:1080")
|
||||
res, err := c.Send(context.Background(), SetParamsUrl("https://www.google.com"), SetParamsMethod(MethodGet))
|
||||
t.Log(string(res), err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
# HTTP连接复用最佳实践
|
||||
|
||||
## 什么是连接复用?
|
||||
|
||||
HTTP连接复用(Connection Reuse)是指在同一个HTTP客户端实例中,对相同目标主机的多个请求复用已建立的TCP连接,而不是为每个请求都创建新的连接。这可以显著提高性能并减少资源消耗。
|
||||
|
||||
## 为什么需要连接复用?
|
||||
|
||||
1. **性能提升**:避免重复的TCP三次握手和TLS握手
|
||||
2. **资源节约**:减少系统文件描述符和内存使用
|
||||
3. **降低延迟**:复用已建立的连接减少连接建立时间
|
||||
4. **服务器友好**:减少服务器连接压力
|
||||
|
||||
## curlx中的连接复用配置
|
||||
|
||||
### 基本配置参数
|
||||
|
||||
```go
|
||||
client := NewCurlx(
|
||||
// 连接池大小配置
|
||||
WithMaxIdleConns(100), // 总空闲连接数上限
|
||||
WithMaxIdleConnsPerHost(10), // 每个主机的空闲连接数
|
||||
WithMaxConnsPerHost(50), // 每个主机的最大连接数
|
||||
WithIdleConnTimeout(90*time.Second), // 空闲连接超时时间
|
||||
|
||||
// 其他优化配置
|
||||
SetOptionTimeOut(30*time.Second),
|
||||
)
|
||||
```
|
||||
|
||||
### 参数详解
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `MaxIdleConns` | 100 | 连接池中保持的最大空闲连接总数 |
|
||||
| `MaxIdleConnsPerHost` | 10 | 对每个主机保持的最大空闲连接数 |
|
||||
| `MaxConnsPerHost` | 50 | 对每个主机允许的最大并发连接数 |
|
||||
| `IdleConnTimeout` | 90s | 空闲连接的超时时间 |
|
||||
|
||||
## 最佳实践
|
||||
|
||||
### 1. 正确使用单例模式
|
||||
|
||||
```go
|
||||
// ❌ 错误做法:每次请求都创建新客户端
|
||||
func badExample() {
|
||||
for i := 0; i < 100; i++ {
|
||||
client := NewCurlx() // 每次都新建,无法复用连接
|
||||
client.Get(context.Background(), "https://example.com")
|
||||
}
|
||||
}
|
||||
|
||||
// ✅ 正确做法:复用客户端实例
|
||||
func goodExample() {
|
||||
client := NewCurlx( // 只创建一次
|
||||
WithMaxIdleConns(50),
|
||||
WithMaxIdleConnsPerHost(5),
|
||||
)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
client.Get(context.Background(), "https://example.com") // 复用连接
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. 合理设置连接池大小
|
||||
|
||||
```go
|
||||
// 根据应用场景调整配置
|
||||
func getConfigForScenario(scenario string) []Option {
|
||||
switch scenario {
|
||||
case "high_concurrency":
|
||||
return []Option{
|
||||
WithMaxIdleConns(200),
|
||||
WithMaxIdleConnsPerHost(20),
|
||||
WithMaxConnsPerHost(100),
|
||||
}
|
||||
case "low_resource":
|
||||
return []Option{
|
||||
WithMaxIdleConns(20),
|
||||
WithMaxIdleConnsPerHost(2),
|
||||
WithMaxConnsPerHost(10),
|
||||
}
|
||||
default:
|
||||
return []Option{
|
||||
WithMaxIdleConns(100),
|
||||
WithMaxIdleConnsPerHost(10),
|
||||
WithMaxConnsPerHost(50),
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. 监控连接池状态
|
||||
|
||||
```go
|
||||
func monitorConnectionPool(client *Curlx) {
|
||||
manager := &ConnectionPoolManager{
|
||||
client: client,
|
||||
transport: client.transport,
|
||||
}
|
||||
|
||||
// 定期检查连接池状态
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
manager.PrintPoolStats()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 性能对比测试
|
||||
|
||||
```go
|
||||
func BenchmarkConnectionReuse(b *testing.B) {
|
||||
client := NewCurlx(
|
||||
WithMaxIdleConns(50),
|
||||
WithMaxIdleConnsPerHost(10),
|
||||
)
|
||||
|
||||
b.Run("with_reuse", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
client.Get(context.Background(), "https://httpbin.org/get")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkWithoutReuse(b *testing.B) {
|
||||
b.Run("without_reuse", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
client := NewCurlx() // 每次新建客户端
|
||||
client.Get(context.Background(), "https://httpbin.org/get")
|
||||
}
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
## 常见问题解答
|
||||
|
||||
### Q: 连接池满了怎么办?
|
||||
A: 当连接池满时,新的请求会等待空闲连接。可以通过增加`MaxConnsPerHost`来缓解。
|
||||
|
||||
### Q: 如何清理空闲连接?
|
||||
A: 空闲连接会在`IdleConnTimeout`后自动关闭,也可以手动调用`transport.CloseIdleConnections()`。
|
||||
|
||||
### Q: 不同主机的连接是否共享?
|
||||
A: 不同主机的连接是隔离的,每个主机维护自己的连接池。
|
||||
|
||||
### Q: HTTPS连接也能复用吗?
|
||||
A: 是的,HTTPS连接同样支持复用,包括TLS会话复用。
|
||||
|
||||
## 调试技巧
|
||||
|
||||
```go
|
||||
// 启用详细的HTTP跟踪
|
||||
import "net/http/httptrace"
|
||||
|
||||
func debugWithTrace() {
|
||||
trace := &httptrace.ClientTrace{
|
||||
GotConn: func(info httptrace.GotConnInfo) {
|
||||
fmt.Printf("连接复用: %v, 来自空闲池: %v\n",
|
||||
info.Reused, info.WasIdle)
|
||||
},
|
||||
ConnectStart: func(network, addr string) {
|
||||
fmt.Printf("开始连接: %s %s\n", network, addr)
|
||||
},
|
||||
ConnectDone: func(network, addr string, err error) {
|
||||
if err != nil {
|
||||
fmt.Printf("连接完成: %v\n", err)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
ctx := httptrace.WithClientTrace(context.Background(), trace)
|
||||
client := NewCurlx()
|
||||
client.Get(ctx, "https://httpbin.org/get")
|
||||
}
|
||||
```
|
||||
|
||||
## 生产环境建议
|
||||
|
||||
1. **预热连接**:应用启动时进行连接预热
|
||||
2. **监控指标**:监控连接池使用率和错误率
|
||||
3. **优雅关闭**:应用关闭时清理连接资源
|
||||
4. **负载均衡**:考虑使用连接池配合负载均衡
|
||||
|
||||
```go
|
||||
// 生产环境推荐配置
|
||||
func productionConfig() *Curlx {
|
||||
return NewCurlx(
|
||||
WithMaxIdleConns(200),
|
||||
WithMaxIdleConnsPerHost(20),
|
||||
WithMaxConnsPerHost(100),
|
||||
WithIdleConnTimeout(120*time.Second),
|
||||
SetOptionTimeOut(30*time.Second),
|
||||
)
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,190 @@
|
||||
package example
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/yuninks/curlx"
|
||||
)
|
||||
|
||||
// ConnectionPoolManager 连接池管理器
|
||||
type ConnectionPoolManager struct {
|
||||
client *curlx.Curlx
|
||||
transport *http.Transport
|
||||
}
|
||||
|
||||
// NewConnectionPoolManager 创建连接池管理器
|
||||
func NewConnectionPoolManager(opts ...curlx.Option) *ConnectionPoolManager {
|
||||
// 添加连接池优化配置
|
||||
poolOpts := append(opts,
|
||||
WithConnectionPoolSettings(
|
||||
100, // MaxIdleConns
|
||||
10, // MaxIdleConnsPerHost
|
||||
50, // MaxConnsPerHost
|
||||
90*time.Second, // IdleConnTimeout
|
||||
),
|
||||
)
|
||||
|
||||
client := curlx.NewCurlx(poolOpts...)
|
||||
|
||||
return &ConnectionPoolManager{
|
||||
client: client,
|
||||
// transport: client.transport,
|
||||
}
|
||||
}
|
||||
|
||||
// WithConnectionPoolSettings 连接池配置选项
|
||||
func WithConnectionPoolSettings(
|
||||
maxIdleConns int,
|
||||
maxIdleConnsPerHost int,
|
||||
maxConnsPerHost int,
|
||||
idleConnTimeout time.Duration,
|
||||
) curlx.Option {
|
||||
return func(options *curlx.ClientOptions) {
|
||||
options.MaxIdleConns = maxIdleConns
|
||||
options.MaxIdleConnsPerHost = maxIdleConnsPerHost
|
||||
options.MaxConnsPerHost = maxConnsPerHost
|
||||
options.IdleConnTimeout = idleConnTimeout
|
||||
}
|
||||
}
|
||||
|
||||
// ConcurrentRequests 并发请求演示
|
||||
func (cpm *ConnectionPoolManager) ConcurrentRequests(urls []string) {
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan string, len(urls))
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
// 并发执行多个请求
|
||||
for i, url := range urls {
|
||||
wg.Add(1)
|
||||
go func(index int, targetURL string) {
|
||||
defer wg.Done()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
response, err := cpm.client.Get(ctx, targetURL)
|
||||
duration := time.Since(start)
|
||||
|
||||
if err != nil {
|
||||
results <- fmt.Sprintf("Request %d to %s failed: %v (took %v)",
|
||||
index, targetURL, err, duration)
|
||||
} else {
|
||||
results <- fmt.Sprintf("Request %d to %s succeeded: %d bytes (took %v)",
|
||||
index, targetURL, len(response), duration)
|
||||
}
|
||||
}(i, url)
|
||||
}
|
||||
|
||||
// 等待所有请求完成
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
totalDuration := time.Since(startTime)
|
||||
fmt.Printf("=== 并发请求完成 ===\n")
|
||||
fmt.Printf("总耗时: %v\n", totalDuration)
|
||||
fmt.Printf("平均每个请求: %v\n", totalDuration/time.Duration(len(urls)))
|
||||
|
||||
// 输出结果
|
||||
for result := range results {
|
||||
fmt.Println(result)
|
||||
}
|
||||
|
||||
// 输出连接池状态
|
||||
cpm.PrintPoolStats()
|
||||
}
|
||||
|
||||
// PrintPoolStats 打印连接池统计信息
|
||||
func (cpm *ConnectionPoolManager) PrintPoolStats() {
|
||||
// stats := cpm.transport
|
||||
|
||||
fmt.Printf("\n=== 连接池统计 ===\n")
|
||||
// fmt.Printf("当前空闲连接数: %d\n", stats.IdleConnCount())
|
||||
// fmt.Printf("总连接数: %d\n", stats.TotalConnCount())
|
||||
// fmt.Printf("等待队列长度: %d\n", stats.WaitQueueLength())
|
||||
}
|
||||
|
||||
// ReuseExample 连接复用示例
|
||||
func (cpm *ConnectionPoolManager) ReuseExample(baseURL string, requestCount int) {
|
||||
fmt.Printf("=== 连接复用测试 ===\n")
|
||||
fmt.Printf("目标URL: %s\n", baseURL)
|
||||
fmt.Printf("请求次数: %d\n\n", requestCount)
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
for i := 0; i < requestCount; i++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
|
||||
start := time.Now()
|
||||
response, err := cpm.client.Get(ctx, baseURL)
|
||||
duration := time.Since(start)
|
||||
|
||||
cancel() // 释放context资源
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("第%d次请求失败: %v (耗时: %v)\n", i+1, err, duration)
|
||||
} else {
|
||||
fmt.Printf("第%d次请求成功: %d字节 (耗时: %v)\n", i+1, len(response), duration)
|
||||
}
|
||||
|
||||
// 小间隔避免请求过于密集
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
totalDuration := time.Since(startTime)
|
||||
fmt.Printf("\n=== 测试完成 ===\n")
|
||||
fmt.Printf("总耗时: %v\n", totalDuration)
|
||||
fmt.Printf("平均每请求: %v\n", totalDuration/time.Duration(requestCount))
|
||||
|
||||
cpm.PrintPoolStats()
|
||||
}
|
||||
|
||||
// PersistentConnectionExample 持久连接示例
|
||||
func (cpm *ConnectionPoolManager) PersistentConnectionExample(targetURL string) {
|
||||
fmt.Printf("=== 持久连接测试 ===\n")
|
||||
fmt.Printf("测试URL: %s\n\n", targetURL)
|
||||
|
||||
// 预热连接
|
||||
fmt.Println("预热阶段 - 建立初始连接...")
|
||||
ctx1, cancel1 := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
_, err := cpm.client.Get(ctx1, targetURL)
|
||||
cancel1()
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("预热失败: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
cpm.PrintPoolStats()
|
||||
|
||||
// 实际测试
|
||||
fmt.Println("\n实际测试阶段...")
|
||||
testStart := time.Now()
|
||||
|
||||
for i := 1; i <= 5; i++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
|
||||
start := time.Now()
|
||||
response, err := cpm.client.Get(ctx, targetURL)
|
||||
duration := time.Since(start)
|
||||
|
||||
cancel()
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("第%d次请求: 失败 (%v) - 耗时: %v\n", i, err, duration)
|
||||
} else {
|
||||
fmt.Printf("第%d次请求: 成功 (%d字节) - 耗时: %v\n", i, len(response), duration)
|
||||
}
|
||||
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
totalTime := time.Since(testStart)
|
||||
fmt.Printf("\n持久连接测试完成,总耗时: %v\n", totalTime)
|
||||
cpm.PrintPoolStats()
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package example
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/yuninks/curlx"
|
||||
)
|
||||
|
||||
func TestConnectionReuse(t *testing.T) {
|
||||
// 创建带优化连接池配置的客户端
|
||||
client := curlx.NewCurlx(
|
||||
curlx.WithMaxIdleConns(50),
|
||||
curlx.WithMaxIdleConnsPerHost(10),
|
||||
curlx.WithMaxConnsPerHost(20),
|
||||
curlx.WithIdleConnTimeout(60*time.Second),
|
||||
curlx.WithOptionTimeOut(30*time.Second),
|
||||
)
|
||||
|
||||
// 测试同一个主机的多次请求,观察连接复用效果
|
||||
targetURL := "https://httpbin.org/get"
|
||||
|
||||
fmt.Println("=== 连接复用测试开始 ===")
|
||||
|
||||
// 预热连接
|
||||
fmt.Println("1. 预热连接...")
|
||||
ctx1, cancel1 := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
_, err := client.Get(ctx1, targetURL)
|
||||
cancel1()
|
||||
if err != nil {
|
||||
t.Logf("预热请求失败: %v", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" 预热完成")
|
||||
|
||||
// 连续请求测试
|
||||
fmt.Println("\n2. 连续请求测试...")
|
||||
for i := 1; i <= 5; i++ {
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
|
||||
response, err := client.Get(ctx, targetURL)
|
||||
duration := time.Since(start)
|
||||
|
||||
cancel()
|
||||
|
||||
if err != nil {
|
||||
t.Logf("第%d次请求失败: %v (耗时: %v)", i, err, duration)
|
||||
} else {
|
||||
t.Logf("第%d次请求成功: %d字节 (耗时: %v)", i, len(response), duration)
|
||||
}
|
||||
|
||||
time.Sleep(200 * time.Millisecond) // 短暂间隔
|
||||
}
|
||||
|
||||
fmt.Println("\n=== 测试完成 ===")
|
||||
}
|
||||
|
||||
func TestConcurrentConnectionReuse(t *testing.T) {
|
||||
// 创建连接池管理器
|
||||
poolManager := NewConnectionPoolManager(
|
||||
curlx.WithMaxIdleConns(100),
|
||||
curlx.WithMaxIdleConnsPerHost(20),
|
||||
curlx.WithMaxConnsPerHost(30),
|
||||
curlx.WithIdleConnTimeout(120*time.Second),
|
||||
)
|
||||
|
||||
// 测试并发请求
|
||||
urls := []string{
|
||||
"https://httpbin.org/get",
|
||||
"https://httpbin.org/uuid",
|
||||
"https://httpbin.org/user-agent",
|
||||
"https://httpbin.org/headers",
|
||||
"https://httpbin.org/ip",
|
||||
}
|
||||
|
||||
fmt.Println("=== 并发连接复用测试 ===")
|
||||
poolManager.ConcurrentRequests(urls)
|
||||
}
|
||||
|
||||
func TestPersistentConnection(t *testing.T) {
|
||||
// 创建优化的客户端
|
||||
client := curlx.NewCurlx(
|
||||
curlx.WithMaxIdleConns(30),
|
||||
curlx.WithMaxIdleConnsPerHost(5),
|
||||
curlx.WithMaxConnsPerHost(15),
|
||||
curlx.WithIdleConnTimeout(30*time.Second),
|
||||
)
|
||||
|
||||
manager := &ConnectionPoolManager{
|
||||
client: client,
|
||||
// transport: client.transport,
|
||||
}
|
||||
|
||||
fmt.Println("=== 持久连接测试 ===")
|
||||
manager.PersistentConnectionExample("https://httpbin.org/delay/1")
|
||||
}
|
||||
|
||||
func Example_connectionReuse() {
|
||||
// 最佳实践示例:如何正确配置连接复用
|
||||
|
||||
// 1. 创建优化配置的客户端
|
||||
client := curlx.NewCurlx(
|
||||
// 连接池配置
|
||||
curlx.WithMaxIdleConns(100), // 总空闲连接数
|
||||
curlx.WithMaxIdleConnsPerHost(10), // 每主机空闲连接数
|
||||
curlx.WithMaxConnsPerHost(50), // 每主机最大连接数
|
||||
curlx.WithIdleConnTimeout(90*time.Second), // 空闲超时时间
|
||||
|
||||
// 其他优化配置
|
||||
curlx.WithOptionTimeOut(30*time.Second),
|
||||
)
|
||||
|
||||
// 2. 复用同一个客户端实例进行多次请求
|
||||
ctx := context.Background()
|
||||
|
||||
// 第一次请求会建立新连接
|
||||
response1, err := client.Get(ctx, "https://httpbin.org/get")
|
||||
if err != nil {
|
||||
fmt.Printf("首次请求失败: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("首次请求成功: %d字节\n", len(response1))
|
||||
|
||||
// 后续请求会复用已有连接
|
||||
response2, err := client.Get(ctx, "https://httpbin.org/uuid")
|
||||
if err != nil {
|
||||
fmt.Printf("第二次请求失败: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("第二次请求成功: %d字节\n", len(response2))
|
||||
|
||||
// 3. 查看连接池状态
|
||||
manager := &ConnectionPoolManager{
|
||||
client: client,
|
||||
// transport: client.transport,
|
||||
}
|
||||
manager.PrintPoolStats()
|
||||
|
||||
// Output:
|
||||
// 首次请求成功: [字节数]
|
||||
// 第二次请求成功: [字节数]
|
||||
// === 连接池统计 ===
|
||||
// 当前空闲连接数: 1
|
||||
// 总连接数: 1
|
||||
// 等待队列长度: 0
|
||||
}
|
||||
@@ -3,6 +3,7 @@ module github.com/yuninks/curlx
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
code.yun.ink/pkg/convx v1.0.3
|
||||
github.com/tidwall/gjson v1.17.0
|
||||
golang.org/x/net v0.18.0
|
||||
)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
code.yun.ink/pkg/convx v1.0.3 h1:pH8dUOgsoaBYVQ3+4C2+uVua561nDxq6/GpaQ9wnCew=
|
||||
code.yun.ink/pkg/convx v1.0.3/go.mod h1:6xqmUend1kwarRvJ0TQlfzzS4QCWdRrXQiUY/ggzYqo=
|
||||
github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
|
||||
github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
|
||||
+73
-19
@@ -6,35 +6,55 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type clientOptions struct {
|
||||
TimeOut time.Duration
|
||||
InsecureSkipVerify bool
|
||||
Logger OptionLogger
|
||||
type ClientOptions struct {
|
||||
TimeOut time.Duration
|
||||
InsecureSkipVerify bool
|
||||
Logger OptionLogger
|
||||
LoggerLength int // 日志输出长度
|
||||
CertFingerprint string // 证书指纹验证
|
||||
|
||||
// 连接池配置
|
||||
MaxIdleConns int
|
||||
MaxIdleConnsPerHost int
|
||||
MaxConnsPerHost int
|
||||
IdleConnTimeout time.Duration
|
||||
}
|
||||
|
||||
func defaultOptions() clientOptions {
|
||||
return clientOptions{
|
||||
TimeOut: time.Second * 120, // 默认超时120
|
||||
Logger: defaultLogger{},
|
||||
func defaultOptions() ClientOptions {
|
||||
return ClientOptions{
|
||||
TimeOut: time.Second * 120, // 默认超时120秒
|
||||
Logger: defaultLogger{},
|
||||
LoggerLength: 100,
|
||||
MaxIdleConns: 100, // 默认连接池大小
|
||||
MaxIdleConnsPerHost: 10, // 每主机默认空闲连接数
|
||||
MaxConnsPerHost: 50, // 每主机最大连接数
|
||||
IdleConnTimeout: 90 * time.Second, // 空闲连接超时
|
||||
}
|
||||
}
|
||||
|
||||
type Option func(*clientOptions)
|
||||
type Option func(*ClientOptions)
|
||||
|
||||
/**
|
||||
* 设置超时时间
|
||||
*/
|
||||
func SetOptionTimeOut(t time.Duration) Option {
|
||||
return func(options *clientOptions) {
|
||||
func WithOptionTimeOut(t time.Duration) Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.TimeOut = t
|
||||
}
|
||||
}
|
||||
|
||||
// 添加证书指纹验证选项
|
||||
func WithOptionTLSPin(certFingerprint string) Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.CertFingerprint = certFingerprint
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 不校验HTTPS证书
|
||||
*/
|
||||
func SetOptionTLSInsecureSkipVerify() Option {
|
||||
return func(options *clientOptions) {
|
||||
func WithOptionTLSInsecureSkipVerify() Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.InsecureSkipVerify = true
|
||||
}
|
||||
}
|
||||
@@ -42,25 +62,59 @@ func SetOptionTLSInsecureSkipVerify() Option {
|
||||
/**
|
||||
* 设置日志输出
|
||||
*/
|
||||
func SetOptionLog(log OptionLogger) Option {
|
||||
return func(options *clientOptions) {
|
||||
func WithOptionLog(log OptionLogger) Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.Logger = log
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置日志输出长度
|
||||
*/
|
||||
func WithOptionLoggerLength(length int) Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.LoggerLength = length
|
||||
}
|
||||
}
|
||||
|
||||
// 连接池配置选项
|
||||
func WithMaxIdleConns(maxIdleConns int) Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.MaxIdleConns = maxIdleConns
|
||||
}
|
||||
}
|
||||
|
||||
func WithMaxIdleConnsPerHost(maxIdleConnsPerHost int) Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.MaxIdleConnsPerHost = maxIdleConnsPerHost
|
||||
}
|
||||
}
|
||||
|
||||
func WithMaxConnsPerHost(maxConnsPerHost int) Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.MaxConnsPerHost = maxConnsPerHost
|
||||
}
|
||||
}
|
||||
|
||||
func WithIdleConnTimeout(timeout time.Duration) Option {
|
||||
return func(options *ClientOptions) {
|
||||
options.IdleConnTimeout = timeout
|
||||
}
|
||||
}
|
||||
|
||||
type OptionLogger interface {
|
||||
Infof(ctx context.Context, format string, args ...interface{})
|
||||
Errorf(ctx context.Context, format string, args ...interface{})
|
||||
Infof(ctx context.Context, format string, args ...any)
|
||||
Errorf(ctx context.Context, format string, args ...any)
|
||||
}
|
||||
|
||||
type defaultLogger struct{}
|
||||
|
||||
func (defaultLogger) Errorf(ctx context.Context, format string, args ...interface{}) {
|
||||
func (defaultLogger) Errorf(ctx context.Context, format string, args ...any) {
|
||||
// 输出日志
|
||||
log.Printf(format, args...)
|
||||
}
|
||||
|
||||
func (defaultLogger) Infof(ctx context.Context, format string, args ...interface{}) {
|
||||
func (defaultLogger) Infof(ctx context.Context, format string, args ...any) {
|
||||
// 输出日志
|
||||
log.Printf(format, args...)
|
||||
}
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
package curlx
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type ClientParams struct {
|
||||
Url string
|
||||
Method Method // GET/POST
|
||||
Body interface{}
|
||||
Headers map[string]interface{}
|
||||
Cookies interface{}
|
||||
Method Method // GET/POST/PUT/DELETE
|
||||
Body []byte
|
||||
Headers http.Header
|
||||
Cookies []http.Cookie
|
||||
ContentType ContentType // FORM,JSON,XML
|
||||
}
|
||||
|
||||
func defaultParams() ClientParams {
|
||||
return ClientParams{
|
||||
Headers: map[string]interface{}{}, // 初始化map
|
||||
Headers: http.Header{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,9 +54,22 @@ func SetParamsMethod(m Method) Param {
|
||||
/**
|
||||
* 设置参数
|
||||
*/
|
||||
func SetParamsBody(p interface{}) Param {
|
||||
func SetParamsBody(by []byte) Param {
|
||||
return func(param *ClientParams) {
|
||||
param.Body = p
|
||||
param.Body = by
|
||||
}
|
||||
}
|
||||
|
||||
func SetParamsBodyAny(v interface{}) Param {
|
||||
return func(param *ClientParams) {
|
||||
switch value := v.(type) {
|
||||
case []byte:
|
||||
param.Body = value
|
||||
case string:
|
||||
param.Body = []byte(value)
|
||||
default:
|
||||
param.Body, _ = json.Marshal(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,12 +78,18 @@ func SetParamsBody(p interface{}) Param {
|
||||
*/
|
||||
func SetParamsFormText(fieldName, fieldValue string) Param {
|
||||
return func(param *ClientParams) {
|
||||
fp := param.Body.([]FormParam)
|
||||
fp = append(fp, FormParam{
|
||||
m := []FormParam{}
|
||||
if param.Body != nil {
|
||||
json.Unmarshal(param.Body, &m)
|
||||
}
|
||||
m = append(m, FormParam{
|
||||
FieldName: fieldName,
|
||||
FieldValue: fieldValue,
|
||||
FieldType: FieldTypeText,
|
||||
})
|
||||
|
||||
fp, _ := json.Marshal(m)
|
||||
|
||||
param.Body = fp
|
||||
}
|
||||
}
|
||||
@@ -75,25 +99,49 @@ func SetParamsFormText(fieldName, fieldValue string) Param {
|
||||
*/
|
||||
func SetParamsFormFile(fieldName, fileName string, fileBytes []byte) Param {
|
||||
return func(param *ClientParams) {
|
||||
fp := param.Body.([]FormParam)
|
||||
|
||||
fp := []FormParam{}
|
||||
|
||||
if param.Body != nil {
|
||||
json.Unmarshal(param.Body, &fp)
|
||||
}
|
||||
|
||||
fp = append(fp, FormParam{
|
||||
FieldName: fieldName,
|
||||
FieldType: FieldTypeFile,
|
||||
FileName: fileName,
|
||||
FileBytes: fileBytes,
|
||||
})
|
||||
param.Body = fp
|
||||
|
||||
fpb, _ := json.Marshal(fp)
|
||||
|
||||
param.Body = fpb
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置请求头
|
||||
*/
|
||||
func SetParamsHeaders(h map[string]interface{}) Param {
|
||||
func SetParamsHeaders(h map[string]string) Param {
|
||||
return func(param *ClientParams) {
|
||||
for key, _ := range h {
|
||||
param.Headers[key] = h[key]
|
||||
if param.Headers == nil {
|
||||
param.Headers = http.Header{}
|
||||
}
|
||||
for k, v := range h {
|
||||
param.Headers.Set(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置请求头
|
||||
*/
|
||||
func SetParamsHeader(key, value string) Param {
|
||||
return func(param *ClientParams) {
|
||||
if param.Headers == nil {
|
||||
param.Headers = http.Header{}
|
||||
}
|
||||
param.Headers.Add(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,7 +150,25 @@ func SetParamsHeaders(h map[string]interface{}) Param {
|
||||
*/
|
||||
func SetUserAgent(userAgent UserAgent) Param {
|
||||
return func(param *ClientParams) {
|
||||
param.Headers["User-Agent"] = string(userAgent)
|
||||
if param.Headers == nil {
|
||||
param.Headers = http.Header{}
|
||||
}
|
||||
param.Headers.Set("User-Agent", string(userAgent))
|
||||
}
|
||||
}
|
||||
|
||||
func SetCookie(name, value string) Param {
|
||||
return func(param *ClientParams) {
|
||||
param.Cookies = append(param.Cookies, http.Cookie{
|
||||
Name: name,
|
||||
Value: value,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func SetCookies(cookies []http.Cookie) Param {
|
||||
return func(param *ClientParams) {
|
||||
param.Cookies = append(param.Cookies, cookies...)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,14 +177,17 @@ func SetUserAgent(userAgent UserAgent) Param {
|
||||
*/
|
||||
func SetReferer(referer string) Param {
|
||||
return func(param *ClientParams) {
|
||||
param.Headers["Referer"] = referer
|
||||
if param.Headers == nil {
|
||||
param.Headers = http.Header{}
|
||||
}
|
||||
param.Headers.Set("Referer", referer)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置cookies
|
||||
*/
|
||||
func SetParamsCookies(c interface{}) Param {
|
||||
func SetParamsCookies(c []http.Cookie) Param {
|
||||
return func(param *ClientParams) {
|
||||
param.Cookies = c
|
||||
}
|
||||
|
||||
+79
-188
@@ -3,14 +3,14 @@ package curlx
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"code.yun.ink/pkg/convx"
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -38,25 +38,13 @@ func (p *ClientParams) parseUrl() error {
|
||||
* 处理请求头Header
|
||||
*/
|
||||
func (p *ClientParams) parseHeaders(r *http.Request) {
|
||||
if p.Headers != nil {
|
||||
if r.Header.Get("User-Agent") == "" {
|
||||
r.Header.Add("User-Agent", string(UserAgentChrome))
|
||||
}
|
||||
for k, v := range p.Headers {
|
||||
switch value := v.(type) {
|
||||
case string:
|
||||
r.Header.Set(k, value)
|
||||
case []string:
|
||||
for _, vv := range value {
|
||||
r.Header.Add(k, vv)
|
||||
}
|
||||
case ContentType:
|
||||
r.Header.Set(k, string(value))
|
||||
case UserAgent:
|
||||
r.Header.Set(k, string(value))
|
||||
}
|
||||
}
|
||||
|
||||
if p.Headers.Get("User-Agent") == "" {
|
||||
p.Headers.Add("User-Agent", string(UserAgentChrome))
|
||||
}
|
||||
|
||||
r.Header = p.Headers
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -67,165 +55,83 @@ func (p *ClientParams) parseParams() (str io.Reader, err error) {
|
||||
|
||||
// 初始化(如未初始化)
|
||||
if p.Headers == nil {
|
||||
p.Headers = make(map[string]interface{})
|
||||
p.Headers = http.Header{}
|
||||
}
|
||||
|
||||
if p.Body != nil {
|
||||
if p.ContentType == ContentTypeJson {
|
||||
// 判断是否存在
|
||||
if _, ok := p.Headers["Content-Type"]; !ok {
|
||||
p.Headers["Content-Type"] = ContentTypeJson
|
||||
}
|
||||
strParam, ok := p.Body.(string)
|
||||
if ok {
|
||||
return bytes.NewReader([]byte(strParam)), nil
|
||||
}
|
||||
b, err := json.Marshal(p.Body)
|
||||
if err == nil {
|
||||
return bytes.NewReader(b), nil
|
||||
}
|
||||
} else if p.ContentType == ContentTypeForm {
|
||||
// 表单上传(可能有文件)
|
||||
// 文件上传的
|
||||
params := []FormParam{}
|
||||
if value, ok := p.Body.([]FormParam); ok {
|
||||
params = value
|
||||
} else if value, ok := p.Body.(FormParam); ok {
|
||||
params = append(params, value)
|
||||
// 添加Content-Type
|
||||
if _, ok := p.Headers["Content-Type"]; !ok {
|
||||
p.Headers.Set("Content-Type", string(p.ContentType))
|
||||
}
|
||||
|
||||
if len(p.Body) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
switch p.ContentType {
|
||||
case ContentTypeJson:
|
||||
// JSON
|
||||
return bytes.NewReader(p.Body), nil
|
||||
case ContentTypeForm:
|
||||
// 表单
|
||||
params := []FormParam{}
|
||||
err = json.Unmarshal(p.Body, ¶ms)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
for _, v := range params {
|
||||
if v.FieldType == FieldTypeFile {
|
||||
part, _ := writer.CreateFormFile(v.FieldName, v.FileName)
|
||||
io.Copy(part, bytes.NewBuffer(v.FileBytes))
|
||||
} else {
|
||||
return nil, errors.New("表单上传的参数格式不正确")
|
||||
_ = writer.WriteField(v.FieldName, v.FieldValue)
|
||||
}
|
||||
}
|
||||
writer.Close()
|
||||
p.Headers.Set("Content-Type", writer.FormDataContentType())
|
||||
return body, nil
|
||||
case ContentTypeXml:
|
||||
// XML
|
||||
return bytes.NewReader(p.Body), nil
|
||||
case ContentTypeText:
|
||||
// TEXT
|
||||
return bytes.NewReader(p.Body), nil
|
||||
case ContentTypeUrlEncoded:
|
||||
// URL编码
|
||||
m := map[string]any{}
|
||||
if err = json.Unmarshal(p.Body, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values := url.Values{}
|
||||
for k, v := range m {
|
||||
val := convx.ToString(v)
|
||||
values.Set(k, val)
|
||||
}
|
||||
|
||||
return strings.NewReader(values.Encode()), nil
|
||||
default:
|
||||
if p.Method == MethodGet {
|
||||
|
||||
m := map[string]any{}
|
||||
if err = json.Unmarshal(p.Body, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
for _, v := range params {
|
||||
if v.FieldType == FieldTypeFile {
|
||||
part, _ := writer.CreateFormFile(v.FieldName, v.FileName)
|
||||
io.Copy(part, bytes.NewBuffer(v.FileBytes))
|
||||
} else {
|
||||
_ = writer.WriteField(v.FieldName, v.FieldValue)
|
||||
}
|
||||
url, err := url.Parse(p.Url) // 解析URL
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writer.Close()
|
||||
p.Headers["Content-Type"] = writer.FormDataContentType()
|
||||
return body, nil
|
||||
query := url.Query()
|
||||
for k, v := range m {
|
||||
val := convx.ToString(v)
|
||||
query[k] = append(query[k], val)
|
||||
}
|
||||
url.RawQuery = query.Encode()
|
||||
p.Url = url.String()
|
||||
|
||||
} else if p.ContentType == ContentTypeXml {
|
||||
if _, ok := p.Headers["Content-Type"]; !ok {
|
||||
p.Headers["Content-Type"] = ContentTypeXml
|
||||
}
|
||||
var string_data string
|
||||
if value, ok := p.Body.(string); ok {
|
||||
string_data = string(value)
|
||||
} else {
|
||||
var by []byte
|
||||
by, err = xml.Marshal(p.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
string_data = string(by)
|
||||
}
|
||||
return strings.NewReader(string_data), nil
|
||||
} else if p.ContentType == ContentTypeText {
|
||||
if _, ok := p.Headers["Content-Type"]; !ok {
|
||||
p.Headers["Content-Type"] = ContentTypeText
|
||||
}
|
||||
|
||||
var string_data string
|
||||
if value, ok := p.Body.(string); ok {
|
||||
string_data = string(value)
|
||||
} else {
|
||||
err = errors.New("TEXT类型的参数仅支持字符串")
|
||||
return
|
||||
}
|
||||
|
||||
return strings.NewReader(string_data), nil
|
||||
} else if p.ContentType == ContentTypeUrlEncoded {
|
||||
if _, ok := p.Headers["Content-Type"]; !ok {
|
||||
p.Headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||
}
|
||||
|
||||
// 判断需要map[string]interface{}类型
|
||||
paramValue, ok := p.Body.(map[string]interface{})
|
||||
if !ok {
|
||||
return strings.NewReader(""), errors.New("参数需map[string]interface{}")
|
||||
}
|
||||
|
||||
values := url.Values{}
|
||||
for k, v := range paramValue {
|
||||
// 字符串
|
||||
if v_string, ok := v.(string); ok {
|
||||
values.Set(k, v_string)
|
||||
}
|
||||
// 字符串切片
|
||||
if vv, ok := v.([]string); ok {
|
||||
for _, vvv := range vv {
|
||||
values.Add(k+"[]", vvv)
|
||||
}
|
||||
}
|
||||
// int转string
|
||||
if v_int, ok := v.(int); ok {
|
||||
values.Set(k, strconv.Itoa(v_int))
|
||||
}
|
||||
// int64转string
|
||||
if v_int64, ok := v.(int64); ok {
|
||||
values.Set(k, strconv.FormatInt(v_int64, 10))
|
||||
}
|
||||
// float32转string
|
||||
if v_float32, ok := v.(float32); ok {
|
||||
values.Set(k, strconv.FormatFloat(float64(v_float32), 'f', -1, 32))
|
||||
}
|
||||
// float64转string
|
||||
if v_float64, ok := v.(float64); ok {
|
||||
values.Set(k, strconv.FormatFloat(v_float64, 'f', -1, 64))
|
||||
}
|
||||
}
|
||||
return strings.NewReader(values.Encode()), nil
|
||||
} else {
|
||||
// 如果是GET请求
|
||||
if p.Method == MethodGet {
|
||||
// 判断需要map[string]interface{}类型
|
||||
paramValue, ok := p.Body.(map[string]interface{})
|
||||
if !ok {
|
||||
return strings.NewReader(""), errors.New("参数需map[string]interface{}")
|
||||
}
|
||||
// 拼接参数到URL
|
||||
if strings.Contains(p.Url, "?") {
|
||||
p.Url += "&"
|
||||
} else {
|
||||
p.Url += "?"
|
||||
}
|
||||
for k, v := range paramValue {
|
||||
// 字符串
|
||||
if v_string, ok := v.(string); ok {
|
||||
p.Url += k + "=" + v_string + "&"
|
||||
}
|
||||
// 字符串切片
|
||||
if vv, ok := v.([]string); ok {
|
||||
for _, vvv := range vv {
|
||||
p.Url += k + "[]=" + vvv + "&"
|
||||
}
|
||||
}
|
||||
// int转string
|
||||
if v_int, ok := v.(int); ok {
|
||||
p.Url += k + "=" + strconv.Itoa(v_int) + "&"
|
||||
}
|
||||
// int64转string
|
||||
if v_int64, ok := v.(int64); ok {
|
||||
p.Url += k + "=" + strconv.FormatInt(v_int64, 10) + "&"
|
||||
}
|
||||
// float32转string
|
||||
if v_float32, ok := v.(float32); ok {
|
||||
p.Url += k + "=" + strconv.FormatFloat(float64(v_float32), 'f', -1, 32) + "&"
|
||||
}
|
||||
// float64转string
|
||||
if v_float64, ok := v.(float64); ok {
|
||||
p.Url += k + "=" + strconv.FormatFloat(v_float64, 'f', -1, 64) + "&"
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return nil, errors.New("curlx 不支持的数据类型")
|
||||
}
|
||||
return nil, errors.New("curlx 不支持的数据类型")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -236,23 +142,8 @@ func (p *ClientParams) parseParams() (str io.Reader, err error) {
|
||||
* 处理Cookie
|
||||
*/
|
||||
func (p *ClientParams) parseCookies(r *http.Request) {
|
||||
switch p.Cookies.(type) {
|
||||
case string:
|
||||
cookies := p.Cookies.(string)
|
||||
r.Header.Add("Cookie", cookies)
|
||||
case map[string]string:
|
||||
cookies := p.Cookies.(map[string]string)
|
||||
for k, v := range cookies {
|
||||
r.AddCookie(&http.Cookie{
|
||||
Name: k,
|
||||
Value: v,
|
||||
})
|
||||
}
|
||||
case []*http.Cookie:
|
||||
cookies := p.Cookies.([]*http.Cookie)
|
||||
for _, cookie := range cookies {
|
||||
r.AddCookie(cookie)
|
||||
}
|
||||
for _, cookie := range p.Cookies {
|
||||
r.AddCookie(&cookie)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+68
-48
@@ -1,6 +1,8 @@
|
||||
package curlx
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -10,70 +12,77 @@ import (
|
||||
|
||||
// Response response object
|
||||
type Response struct {
|
||||
resp *http.Response
|
||||
req *http.Request
|
||||
body []byte
|
||||
err error
|
||||
Response *http.Response
|
||||
Request *http.Request
|
||||
Body []byte
|
||||
Err error
|
||||
}
|
||||
|
||||
// ResponseBody response body
|
||||
type ResponseBody []byte
|
||||
|
||||
// String fmt outout
|
||||
func (r ResponseBody) String() string {
|
||||
return string(r)
|
||||
}
|
||||
|
||||
// Read get slice of response body
|
||||
func (r ResponseBody) Read(length int) []byte {
|
||||
if length > len(r) {
|
||||
length = len(r)
|
||||
func (l *Response) Close() error {
|
||||
if l.Response.Body != nil {
|
||||
return l.Response.Body.Close()
|
||||
}
|
||||
|
||||
return r[:length]
|
||||
}
|
||||
|
||||
// GetContents format response body as string
|
||||
func (r ResponseBody) GetContents() string {
|
||||
return string(r)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetRequest get request object
|
||||
func (r *Response) GetRequest() *http.Request {
|
||||
return r.req
|
||||
return r.Request
|
||||
}
|
||||
|
||||
func (r *Response) GetResponse() *http.Response {
|
||||
return r.Response
|
||||
}
|
||||
|
||||
// GetBody parse response body
|
||||
func (r *Response) GetBody() (ResponseBody, error) {
|
||||
return ResponseBody(r.body), r.err
|
||||
func (r *Response) GetBody() ([]byte, error) {
|
||||
if r.Err != nil {
|
||||
return nil, r.Err
|
||||
}
|
||||
if r.Body != nil {
|
||||
return r.Body, nil
|
||||
}
|
||||
if r.Response == nil {
|
||||
return nil, nil
|
||||
}
|
||||
body := []byte{}
|
||||
var err error
|
||||
if r.Response.Header.Get("Content-Encoding") == "gzip" {
|
||||
reader, err := gzip.NewReader(r.Response.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
body, err = io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
body, err = io.ReadAll(r.Response.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// close body
|
||||
r.Response.Body.Close()
|
||||
|
||||
r.Body = body
|
||||
return body, err
|
||||
}
|
||||
|
||||
// GetParsedBody parse response body with gjson
|
||||
func (r *Response) GetParsedBody() (*gjson.Result, error) {
|
||||
pb := gjson.ParseBytes(r.body)
|
||||
|
||||
return &pb, nil
|
||||
}
|
||||
|
||||
// GetStatusCode get response status code
|
||||
func (r *Response) GetStatusCode() int {
|
||||
return r.resp.StatusCode
|
||||
}
|
||||
|
||||
// GetReasonPhrase get response reason phrase
|
||||
func (r *Response) GetReasonPhrase() string {
|
||||
status := r.resp.Status
|
||||
arr := strings.Split(status, " ")
|
||||
|
||||
return arr[1]
|
||||
func (r Response) GetStatusCode() int {
|
||||
if r.Response == nil {
|
||||
return 0
|
||||
}
|
||||
return r.Response.StatusCode
|
||||
}
|
||||
|
||||
// IsTimeout get if request is timeout
|
||||
func (r *Response) IsTimeout() bool {
|
||||
if r.err == nil {
|
||||
if r.Err == nil {
|
||||
return false
|
||||
}
|
||||
netErr, ok := r.err.(net.Error)
|
||||
netErr, ok := r.Err.(net.Error)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
@@ -84,16 +93,27 @@ func (r *Response) IsTimeout() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// GetParsedBody parse response body with gjson
|
||||
func (r *Response) GetParsedBody() (*gjson.Result, error) {
|
||||
body, err := r.GetBody()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pb := gjson.ParseBytes(body)
|
||||
|
||||
return &pb, nil
|
||||
}
|
||||
|
||||
// GetHeaders get response headers
|
||||
func (r *Response) GetHeaders() map[string][]string {
|
||||
return r.resp.Header
|
||||
return r.Response.Header
|
||||
}
|
||||
|
||||
// GetHeader get response header
|
||||
func (r *Response) GetHeader(name string) []string {
|
||||
headers := r.GetHeaders()
|
||||
for k, v := range headers {
|
||||
if strings.ToLower(name) == strings.ToLower(k) {
|
||||
if strings.EqualFold(name, k) {
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
package curlx
|
||||
|
||||
|
||||
type UserAgent string
|
||||
|
||||
const(
|
||||
UserAgentChrome UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
UserAgentFirefox UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 "
|
||||
UserAgentIE UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; "
|
||||
UserAgentEdge UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
UserAgentWechat UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
|
||||
)
|
||||
Reference in New Issue
Block a user