From 3545414ef21169888e107b3ea72c22b9c2d8e97f Mon Sep 17 00:00:00 2001 From: Yun Date: Fri, 10 Nov 2023 00:05:27 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- arrayx/arrayx.go | 95 +++++++++++-- arrayx/arrayx_test.go | 6 + cachex/cachex.go | 57 ++++---- cachex/cachex_test.go | 20 +++ configx/viperx/config.yml | 1 + configx/viperx/viperx.go | 15 +- configx/viperx/viperx_test.go | 19 +++ httpx/curlx/curlx.go | 254 ---------------------------------- httpx/curlx/request.go | 218 ----------------------------- httpx/curlx/resopnse.go | 124 ----------------- httpx/post.go | 3 - loggerx/.gitignore | 1 + 12 files changed, 166 insertions(+), 647 deletions(-) create mode 100644 cachex/cachex_test.go create mode 100644 configx/viperx/config.yml create mode 100644 configx/viperx/viperx_test.go delete mode 100644 httpx/curlx/curlx.go delete mode 100644 httpx/curlx/request.go delete mode 100644 httpx/curlx/resopnse.go delete mode 100644 httpx/post.go create mode 100644 loggerx/.gitignore diff --git a/arrayx/arrayx.go b/arrayx/arrayx.go index 412367a..8402a8a 100644 --- a/arrayx/arrayx.go +++ b/arrayx/arrayx.go @@ -1,15 +1,55 @@ package arrayx // Author: yun -// Version: 2023年6月12日18:54:17 +// Version: 2023年10月24日15:31:41 import ( + "fmt" "reflect" - "sort" ) /** * 判断是否在一个数组里面 + * + * 针对需要重复判断的情况,这个方法会比InArray()快一点 + * + * @param target interface{} 目标值 + * @param array interface{} 数组 + * @return bool 是否存在 + * @return int 索引 + */ + +type inArray struct { + array interface{} +} + +func NewArray(array interface{}) *inArray { + return &inArray{array: array} +} +func (i *inArray) In(target interface{}) (exists bool, index int) { + exists = false + index = -1 + switch reflect.TypeOf(i.array).Kind() { + case reflect.Slice: + s := reflect.ValueOf(i.array) + for i := 0; i < s.Len(); i++ { + if reflect.DeepEqual(target, s.Index(i).Interface()) { + index = i + exists = true + return + } + } + } + return +} + +/** + * 判断是否在一个数组里面 + * + * @param target interface{} 目标值 + * @param array interface{} 数组 + * @return bool 是否存在 + * @return int 索引 */ func InArray(target interface{}, array interface{}) (exists bool, index int) { exists = false @@ -30,16 +70,16 @@ func InArray(target interface{}, array interface{}) (exists bool, index int) { // 判断字符串是否在字符串数组中 // TODO:待验证 -func InStringArray(target string, strArray []string) bool { - // 对字符串切片进行排序 - sort.Strings(strArray) - index := sort.SearchStrings(strArray, target) - // 先判断 &&左侧的条件，如果不满足则结束此处判断，不会再进行右侧的判断 - if index < len(strArray) && strArray[index] == target { - return true - } - return false -} +// func InStringArray(target string, strArray []string) bool { +// // 对字符串切片进行排序 +// sort.Strings(strArray) +// index := sort.SearchStrings(strArray, target) +// // 先判断 &&左侧的条件，如果不满足则结束此处判断，不会再进行右侧的判断 +// if index < len(strArray) && strArray[index] == target { +// return true +// } +// return false +// } // 数组去重 func ArrayUniqueString(target []string) []string { @@ -54,6 +94,34 @@ func ArrayUniqueString(target []string) []string { return result } +func ArrayUniqueInt64(target []int64) []int64 { + result := make([]int64, 0, len(target)) + temp := map[int64]struct{}{} + for _, item := range target { + if _, ok := temp[item]; !ok { + temp[item] = struct{}{} + result = append(result, item) + } + } + return result +} + +func ArrayInt64ToString(target []int64) []string { + result := make([]string, 0) + for _, val := range target { + result = append(result, fmt.Sprintf("%d", val)) + } + return result +} + +func ArrayIntToString(target []int) []string { + result := make([]string, 0) + for _, val := range target { + result = append(result, fmt.Sprintf("%d", val)) + } + return result +} + // 交集 // 并集 // 差集 @@ -72,6 +140,3 @@ func ArrayUniqueString(target []string) []string { // 求最大值和最小值的差值的绝对值的平均值的平方根 // 求最大值和最小值的差值的绝对值的平均值的平方根的平方 // 求最大值和最小值的差值的绝对值的平均值的平方根的平方的平均值 - - - diff --git a/arrayx/arrayx_test.go b/arrayx/arrayx_test.go index 3c3c838..252ef59 100644 --- a/arrayx/arrayx_test.go +++ b/arrayx/arrayx_test.go @@ -18,6 +18,12 @@ func TestInArray(t *testing.T) { t.Error("int64类型不应该在int数组里面") } + exist, index = arrayx.NewArray(arr1).In(val) + t.Log("2", exist, index) + if !exist { + t.Error("int64类型不应该在int数组里面") + } + exist, index = arrayx.InArray(val32, arr1) t.Log(exist, index) if exist { diff --git a/cachex/cachex.go b/cachex/cachex.go index 60b5c1a..57b843e 100644 --- a/cachex/cachex.go +++ b/cachex/cachex.go @@ -5,12 +5,16 @@ import ( "time" ) -// 内存缓存 +// 应用内缓存 +// 特点 +// 1.全局单实例 // 到设定的点就删除 -var store sync.Map - -type cache struct {} +type cache struct{ + store sync.Map +} +var one sync.Once +var c cache type cacheData struct { key string @@ -19,20 +23,38 @@ type cacheData struct { } func NewCache() *cache { - return &cache{} + one.Do(func() { + c = cache{} + + go func(){ + for { + c.store.Range(func(key, value interface{}) bool { + if value.(*cacheData).expire.Before(time.Now()) { + c.store.Delete(key) + } + return true + }) + + time.Sleep(time.Second * 5) + } + }() + }) + return &c } +// 设置缓存 func (c *cache) Set(key string, value interface{}, expire time.Duration) { - cd := &cacheData{key,value,time.Now().Add(expire)} - store.Store(key, cd) + cd := &cacheData{key, value, time.Now().Add(expire)} + c.store.Store(key, cd) } +// 读取缓存 func (c *cache) Get(key string) interface{} { - if v, ok := store.Load(key); ok { + if v, ok := c.store.Load(key); ok { cc := v.(*cacheData) if cc.expire.Before(time.Now()) { - store.Delete(key) + c.store.Delete(key) return nil } return cc.data @@ -40,20 +62,7 @@ func (c *cache) Get(key string) interface{} { return nil } +// 删除缓存 func (c *cache) Delete(key string) { - store.Delete(key) -} - -func init() { - for { - store.Range(func(key, value interface{}) bool { - if value.(*cacheData).expire.Before(time.Now()) { - store.Delete(key) - } - return true - }) - - time.Sleep(time.Second * 5) - } - + c.store.Delete(key) } diff --git a/cachex/cachex_test.go b/cachex/cachex_test.go new file mode 100644 index 0000000..8d6352e --- /dev/null +++ b/cachex/cachex_test.go @@ -0,0 +1,20 @@ +package cachex_test + +import ( + "fmt" + "testing" + "time" + + "code.yun.ink/open/utils/cachex" +) + +func TestCache(t *testing.T) { + cachex.NewCache().Set("test", "test", time.Second*5) + + da := cachex.NewCache().Get("test") + fmt.Println(da) + + time.Sleep(time.Second *5) + da = cachex.NewCache().Get("test") + fmt.Println(da) +} diff --git a/configx/viperx/config.yml b/configx/viperx/config.yml new file mode 100644 index 0000000..a1d51a6 --- /dev/null +++ b/configx/viperx/config.yml @@ -0,0 +1 @@ +name: demo \ No newline at end of file diff --git a/configx/viperx/viperx.go b/configx/viperx/viperx.go index 70d35e1..82a0f3b 100644 --- a/configx/viperx/viperx.go +++ b/configx/viperx/viperx.go @@ -8,7 +8,6 @@ import ( "fmt" "log" "os" - "yunink/app/config" "github.com/fsnotify/fsnotify" "github.com/spf13/viper" @@ -16,7 +15,7 @@ import ( // 使用viper初始化配置 -func InitConfig(path string) (*config.Config, error) { +func InitConfig(path string, data interface{}) error { if len(path) == 0 { flag.StringVar(&path, "c", "", "choose config file.") @@ -38,24 +37,22 @@ func InitConfig(path string) (*config.Config, error) { v.SetConfigFile(path) err := v.ReadInConfig() if err != nil { - return nil, err + return err } // 监控配置文件的变化 v.WatchConfig() - resp := new(config.Config) - // 监听配置的变化 v.OnConfigChange(func(e fsnotify.Event) { fmt.Println("config file changed:", e.Name) - if err := v.Unmarshal(&resp); err != nil { + if err := v.Unmarshal(&data); err != nil { log.Println(err) } }) - if err := v.Unmarshal(&resp); err != nil { + if err := v.Unmarshal(&data); err != nil { log.Println(err) - return nil, err + return err } - return resp, nil + return nil } diff --git a/configx/viperx/viperx_test.go b/configx/viperx/viperx_test.go new file mode 100644 index 0000000..33893ad --- /dev/null +++ b/configx/viperx/viperx_test.go @@ -0,0 +1,19 @@ +package viperx + +import ( + "fmt" + "testing" +) + +func TestViper(t *testing.T) { + var data Config + err := InitConfig("./config.yml", &data) + if err != nil { + panic(err) + } + fmt.Println(data) +} + +type Config struct { + Name string `mapstructure:"name" json:"name" yaml:"name"` +} diff --git a/httpx/curlx/curlx.go b/httpx/curlx/curlx.go deleted file mode 100644 index b2ceca6..0000000 --- a/httpx/curlx/curlx.go +++ /dev/null @@ -1,254 +0,0 @@ -package curlx - -import ( - "bufio" - "compress/gzip" - "context" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "time" - - "golang.org/x/net/proxy" -) - -/** - * Author: Yun - * Date: 2023年7月12日11:35:01 - */ - -type dataType string - -const ( - DataTypeForm dataType = "form" - DataTypeJson dataType = "json" - DataTypeXml dataType = "xml" - DataTypeEncode dataType = "encode" - DataTypeText dataType = "text" -) - -type method string - -const ( - MethodGet method = "GET" - MethodPost method = "POST" -) - -type CurlParams struct { - Url string - Method method // GET/POST - Params interface{} - Headers map[string]interface{} - Cookies interface{} - DataType dataType // FORM,JSON,XML -} - -var ( - defaultTimeOut = 180 // 默认(最长超时时间) - // 默认的transport - transport http.Transport = http.Transport{ - // Dial: func(netw, addr string) (net.Conn, error) { - // // 这里指定域名访问的IP - // // if addr == "api.hk.blueoceanpay.com:443" { - // // addr = "47.56.200.21:443" - // // } - // conn, err := net.DialTimeout(netw, addr, time.Second*time.Duration(timeOut)) // 设置建立连接超时 - // if err != nil { - // return nil, err - // } - // // conn.RemoteAddr().String() - // conn.SetDeadline(time.Now().Add(time.Second * time.Duration(timeOut))) // 设置发送接收数据超时 - // return conn, nil - // }, - // ResponseHeaderTimeout: time.Second * time.Duration(defaultTimeOut), // 响应超时 - DisableKeepAlives: false, // 短连接（默认是使用长连接，连接过多时会造成服务器拒绝服务问题） - MaxIdleConns: 0, // 所有host的连接池最大连接数量，默认无穷大 - MaxIdleConnsPerHost: 5, // 每个host的连接池最大空闲连接收，默认2 - MaxConnsPerHost: 0, // 每个host的最大连接数量 - IdleConnTimeout: time.Second * 2, // 空闲连接超时关闭的时间 - } - // client = &http.Client{} - -) - -type DialContext func(ctx context.Context, network, addr string) (net.Conn, error) - -type curlx struct { - transport *http.Transport -} - -func NewCurlx() *curlx { - return &curlx{ - transport: &transport, - } -} - -/** - * 使用Socks5代理 - */ -func (c *curlx) WithSocks5(address string) error { - baseDialer := &net.Dialer{ - Timeout: 180 * time.Second, - KeepAlive: 180 * time.Second, - } - dialSocksProxy, err := proxy.SOCKS5("tcp", address, nil, baseDialer) - if err != nil { - fmt.Println("proxy.SOCKS5 err", err) - return err - } - dialContext := (baseDialer).DialContext - if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok { - dialContext = contextDialer.DialContext - } - c.transport.DialContext = dialContext - return nil -} - -/** - * 不校验HTTPS证书 - */ -func (c *curlx) WithInsecureSkipVerify() { - c.transport.TLSClientConfig.InsecureSkipVerify = true -} - -/** - * 设置超时时间,单位秒 - */ -func (c *curlx) WithTimeout(timeout int) { - c.transport.ResponseHeaderTimeout = time.Second * time.Duration(timeout) -} - -/** - * 简单请求 - */ -func (c *curlx) Send(ctx context.Context, p *CurlParams) (res string, httpcode int, err error) { - response, err := c.sendExec(ctx, p) - if err != nil { - return "", -1, err - } - - defer response.Body.Close() // 处理完关闭 - - // stdout := os.Stdout // 将结果定位到标准输出，也可以直接打印出来，或定位到其他地方进行相应处理 - // _, err = io.Copy(stdout, response.Body) // 将第二个参数拷贝到第一个参数，直到第二参数到达EOF或发生错误，返回拷贝的值 - status := response.StatusCode // 获取状态码，正常是200 - - var body []byte - switch response.Header.Get("Content-Encoding") { - case "gzip": - reader, err := gzip.NewReader(response.Body) - if err != nil { - return "", status, err - } - for { - buf := make([]byte, 1024) - n, err := reader.Read(buf) - if err != nil && err != io.EOF { - panic(err) - } - if n == 0 { - break - } - body = append(body, buf...) - } - default: - body, _ = ioutil.ReadAll(response.Body) - } - return string(body), status, nil -} - -/** - * 执行发送 - * 注意：外部使用需要加这一句 defer response.Body.Close() - */ -func (c *curlx) sendExec(ctx context.Context, p *CurlParams) (resp *http.Response, err error) { - client := &http.Client{ - Timeout: time.Second * time.Duration(defaultTimeOut), // 设置该条连接的超时 - Transport: c.transport, // - } - - err = p.parseMethod() - if err != nil { - return nil, err - } - - // 判断和处理url - err = p.parseUrl() - if err != nil { - return nil, err - } - - // 处理参数 - reqParams, err := p.parseParams() - if err != nil { - return nil, err - } - - // 初始化句柄 - request, err := http.NewRequest( // 提交请求 用指定的方法 - string(p.Method), - p.Url, - reqParams, - ) - if err != nil { - return nil, err - } - - // 这里指定要访问的HOST,到时候服务器获取主机是获取到这个 - // request.Host = "api.hk.blueoceantech.co" - - // 设置上下文控制 - request = request.WithContext(ctx) - - // 处理请求头 - p.parseHeaders(request) - - // 处理Cookies - p.parseCookies(request) - - // 发起请求 - response, err := client.Do(request) - if err != nil { - return nil, err - } - return response, nil -} - -/** - * 流式请求 - */ -func (c *curlx) SendChan(ctx context.Context, p *CurlParams) (<-chan string, error) { - - data := make(chan string, 1000) - - go func() { - defer close(data) - - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) - defer cancel() - - response, err := c.sendExec(ctx, p) - if err != nil { - return - } - defer response.Body.Close() // 处理完关闭 - - scanner := bufio.NewScanner(response.Body) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - // 30min超时 - select { - case <-ctx.Done(): - return - case data <- text: - } - } - }() - - return data, nil -} diff --git a/httpx/curlx/request.go b/httpx/curlx/request.go deleted file mode 100644 index 8ce61e9..0000000 --- a/httpx/curlx/request.go +++ /dev/null @@ -1,218 +0,0 @@ -package curlx - -import ( - "bytes" - "encoding/json" - "encoding/xml" - "errors" - "io" - "net/http" - "net/url" - "strconv" - "strings" -) - -/** - * 处理请求类型 - */ -func (p *CurlParams) parseMethod() error { - if p.Method == "" { - return errors.New("请求类型不能为空") - } - return nil -} - -/** - * 处理URL - */ -func (p *CurlParams) parseUrl() error { - _, err := url.Parse(p.Url) - if err != nil { - return err - } - return nil -} - -/** - * 处理请求头Header - */ -func (p *CurlParams) parseHeaders(r *http.Request) { - if p.Headers != nil { - if r.Header.Get("User-Agent") == "" { - r.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0 Send By Golang") - } - for k, v := range p.Headers { - if vv, ok := v.(string); ok { - r.Header.Set(k, vv) - continue - } - if vv, ok := v.([]string); ok { - for _, vvv := range vv { - r.Header.Add(k, vvv) - } - } - } - } -} - -/** - * 处理请求参数 - */ -func (p *CurlParams) parseParams() (str io.Reader, err error) { - err = nil - - // 初始化(如未初始化) - if p.Headers == nil { - p.Headers = make(map[string]interface{}) - } - - if p.Params != nil { - if p.DataType == DataTypeJson { - // 判断是否存在 - if _, ok := p.Headers["Content-Type"]; !ok { - p.Headers["Content-Type"] = "application/json" - } - strParam, ok := p.Params.(string) - if ok { - return bytes.NewReader([]byte(strParam)), nil - } - b, err := json.Marshal(p.Params) - if err == nil { - return bytes.NewReader(b), nil - } - } else if p.DataType == DataTypeXml { - if _, ok := p.Headers["Content-Type"]; !ok { - p.Headers["Content-Type"] = "application/xml" - } - var string_data string - if value, ok := p.Params.(string); ok { - string_data = string(value) - } else { - var by []byte - by, err = xml.Marshal(p.Params) - if err != nil { - return - } - string_data = string(by) - } - return strings.NewReader(string_data), nil - // switch p.Params.(type) { - // case map[string]string: - // // 请求参数转换成xml结构 - // b, err := goutils.Map2XML(p.Params.(map[string]string)) - // if err == nil { - // return bytes.NewBuffer(b) - // } - // default: - // b, err := xml.Marshal(p.Params) - // if err == nil { - // return bytes.NewBuffer(b) - // } - // } - } else if p.DataType == DataTypeText { - if _, ok := p.Headers["Content-Type"]; !ok { - p.Headers["Content-Type"] = "text/plain" - } - - var string_data string - if value, ok := p.Params.(string); ok { - string_data = string(value) - } else { - err = errors.New("TEXT类型的参数仅支持字符串") - return - } - - return strings.NewReader(string_data), nil - } else { - // FORM,"" - if _, ok := p.Headers["Content-Type"]; !ok { - p.Headers["Content-Type"] = "application/x-www-form-urlencoded" - } - - // 判断需要map[string]interface{}类型 - paramValue, ok := p.Params.(map[string]interface{}) - if !ok { - return strings.NewReader(""), errors.New("参数需map[string]interface{}") - } - - values := url.Values{} - for k, v := range paramValue { - // 字符串 - if v_string, ok := v.(string); ok { - values.Set(k, v_string) - } - // 字符串切片 - if vv, ok := v.([]string); ok { - for _, vvv := range vv { - values.Add(k+"[]", vvv) - } - } - // int转string - if v_int, ok := v.(int); ok { - values.Set(k, strconv.Itoa(v_int)) - } - // int64转string - if v_int64, ok := v.(int64); ok { - values.Set(k, strconv.FormatInt(v_int64, 10)) - } - // float32转string - if v_float32, ok := v.(float32); ok { - values.Set(k, strconv.FormatFloat(float64(v_float32), 'f', -1, 32)) - } - // float64转string - if v_float64, ok := v.(float64); ok { - values.Set(k, strconv.FormatFloat(v_float64, 'f', -1, 64)) - } - } - return strings.NewReader(values.Encode()), nil - } - - } - return -} - -/** - * 处理Cookie - */ -func (p *CurlParams) parseCookies(r *http.Request) { - switch p.Cookies.(type) { - case string: - cookies := p.Cookies.(string) - r.Header.Add("Cookie", cookies) - case map[string]string: - cookies := p.Cookies.(map[string]string) - for k, v := range cookies { - r.AddCookie(&http.Cookie{ - Name: k, - Value: v, - }) - } - case []*http.Cookie: - cookies := p.Cookies.([]*http.Cookie) - for _, cookie := range cookies { - r.AddCookie(cookie) - } - } -} - -// func (r *Request) parseQuery() { -// switch r.opts.Query.(type) { -// case string: -// str := r.opts.Query.(string) -// r.req.URL.RawQuery = str -// case map[string]interface{}: -// q := r.req.URL.Query() -// for k, v := range r.opts.Query.(map[string]interface{}) { -// if vv, ok := v.(string); ok { -// q.Set(k, vv) -// continue -// } -// if vv, ok := v.([]string); ok { -// for _, vvv := range vv { -// q.Add(k, vvv) -// } -// } -// } -// r.req.URL.RawQuery = q.Encode() -// } -// } diff --git a/httpx/curlx/resopnse.go b/httpx/curlx/resopnse.go deleted file mode 100644 index 11c7e0c..0000000 --- a/httpx/curlx/resopnse.go +++ /dev/null @@ -1,124 +0,0 @@ -package curlx - -import ( - "net" - "net/http" - "strings" - - "github.com/tidwall/gjson" -) - -// Response response object -type Response struct { - resp *http.Response - req *http.Request - body []byte - err error -} - -// ResponseBody response body -type ResponseBody []byte - -// String fmt outout -func (r ResponseBody) String() string { - return string(r) -} - -// Read get slice of response body -func (r ResponseBody) Read(length int) []byte { - if length > len(r) { - length = len(r) - } - - return r[:length] -} - -// GetContents format response body as string -func (r ResponseBody) GetContents() string { - return string(r) -} - -// GetRequest get request object -func (r *Response) GetRequest() *http.Request { - return r.req -} - -// GetBody parse response body -func (r *Response) GetBody() (ResponseBody, error) { - return ResponseBody(r.body), r.err -} - -// GetParsedBody parse response body with gjson -func (r *Response) GetParsedBody() (*gjson.Result, error) { - pb := gjson.ParseBytes(r.body) - - return &pb, nil -} - -// GetStatusCode get response status code -func (r *Response) GetStatusCode() int { - return r.resp.StatusCode -} - -// GetReasonPhrase get response reason phrase -func (r *Response) GetReasonPhrase() string { - status := r.resp.Status - arr := strings.Split(status, " ") - - return arr[1] -} - -// IsTimeout get if request is timeout -func (r *Response) IsTimeout() bool { - if r.err == nil { - return false - } - netErr, ok := r.err.(net.Error) - if !ok { - return false - } - if netErr.Timeout() { - return true - } - - return false -} - -// GetHeaders get response headers -func (r *Response) GetHeaders() map[string][]string { - return r.resp.Header -} - -// GetHeader get response header -func (r *Response) GetHeader(name string) []string { - headers := r.GetHeaders() - for k, v := range headers { - if strings.ToLower(name) == strings.ToLower(k) { - return v - } - } - - return nil -} - -// GetHeaderLine get a single response header -func (r *Response) GetHeaderLine(name string) string { - header := r.GetHeader(name) - if len(header) > 0 { - return header[0] - } - - return "" -} - -// HasHeader get if header exsits in response headers -func (r *Response) HasHeader(name string) bool { - headers := r.GetHeaders() - for k := range headers { - if strings.ToLower(name) == strings.ToLower(k) { - return true - } - } - - return false -} diff --git a/httpx/post.go b/httpx/post.go deleted file mode 100644 index 5aa5af3..0000000 --- a/httpx/post.go +++ /dev/null @@ -1,3 +0,0 @@ -package httpx - -func Post(){} diff --git a/loggerx/.gitignore b/loggerx/.gitignore new file mode 100644 index 0000000..fbf828d --- /dev/null +++ b/loggerx/.gitignore @@ -0,0 +1 @@ +log \ No newline at end of file