From 92c76111b83f39adc360c56b955a4551a727f403 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Sun, 7 Feb 2021 13:14:01 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84es=20engine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/elasticEngine.go | 114 +++++++++++----------- test/portData_test.go | 30 +++--- utils/reflect.go | 219 +++++++++++++++++++++++++++--------------- 3 files changed, 214 insertions(+), 149 deletions(-) diff --git a/db/elasticEngine.go b/db/elasticEngine.go index 9aaa9d9..c0e1f05 100644 --- a/db/elasticEngine.go +++ b/db/elasticEngine.go @@ -10,109 +10,107 @@ import ( "gopkg.in/olivere/elastic.v7" "qiniupkg.com/x/log.v7" ) -const( - ERROR_PTR = "null pointer error" - INPUT_TYPE_ERROR = "wrong input parameter" - CREATED_ERROR = "create error" - DELETE_ERROR = "delete error" - INDEX_EXISTED = "index existed" +const ( + ERROR_PTR = "null pointer error" + INPUT_TYPE_ERROR = "wrong input parameter" + CREATED_ERROR = "create error" + DELETE_ERROR = "delete error" + INDEX_EXISTED = "index existed" ) -type ElkEngine struct { + +type ElkEngine struct { cli *elastic.Client } - - -func (p *ElkEngine)Create(index string,types string,id string,data interface{}) (error) { - if nil != p{ - if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct){ +func (p *ElkEngine) Create(index string, types string, id string, data interface{}) error { + if nil != p { + if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct) { resp, err := p.cli.Index(). Index(index). BodyJson(data). Do(context.Background()) if err != nil { - log.Print("create error",err) + log.Print("create error", err) return err } log.Print(resp) - }else{ + } else { log.Print(reflect.TypeOf(data).Kind()) return errors.New(INPUT_TYPE_ERROR) } - }else{ + } else { return errors.New(ERROR_PTR) } return nil } -func (p *ElkEngine)Delete(query elastic.Query,index string) error{ - if nil != p{ +func (p *ElkEngine) Delete(query elastic.Query, index string) error { + if nil != p { _, err := p.cli.DeleteByQuery().Index(index).Query(query). Do(context.Background()) if err != nil { log.Print(err) - return err + return err } - }else{ + } else { return errors.New(ERROR_PTR) } return nil } -/* -*/ -func (p *ElkEngine)Query(index string,query elastic.Query,typ reflect.Type, - limit int,offset int) ([]interface{},[]string,error) { +func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type, + limit int, offset int) ([]interface{}, []string, error) { rets := []interface{}{} - if nil != p{ - if(limit == 0){ + if nil != p { + if limit == 0 { res, err := p.cli.Search(index).Query(query).Do(context.Background()) if err != nil { print(err) - return nil,nil,err + return nil, nil, err } - + id := []string{} - for _,vs := range res.Hits.Hits{ - id = append(id,vs.Id) + for _, vs := range res.Hits.Hits { + id = append(id, vs.Id) } - return rets,id,nil - }else{ - res, err := p.cli.Search(index).Query(query).Size(limit).From(limit*offset).Do(context.Background()) + return rets, id, nil + } else { + res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background()) if err != nil { print(err) - return nil,nil,err + return nil, nil, err } id := []string{} - for _,vs := range res.Hits.Hits{ - id = append(id,vs.Id) - data,e := vs.Source.MarshalJSON() - if nil != e{ + for _, vs := range res.Hits.Hits { + id = append(id, vs.Id) + data, e := vs.Source.MarshalJSON() + if nil != e { log.Print(e.Error()) } - obj := utils.ReflectMakeNew(typ) mapobj := map[string]interface{}{} - e = json.Unmarshal(data,&mapobj) - if nil != e{ + e = json.Unmarshal(data, &mapobj) + if nil != e { + log.Print(e.Error()) + } + obj, e := utils.UnmarshalJson2StructGen(typ, mapobj) + if nil != e { log.Print(e.Error()) } // for k,_ := range mapobj{ // value := reflect.ValueOf(obj) // } - log.Print(obj) - rets = append(rets,obj) - log.Print(string(data)) + rets = append(rets, obj) } - return rets,id,nil + return rets, id, nil } - }else{ - return nil,nil,errors.New(ERROR_PTR) + } else { + return nil, nil, errors.New(ERROR_PTR) } } -func (p *ElkEngine)Update(index string,types string,id string,data map[string]interface{}) error { +func (p *ElkEngine) Update(index string, types string, id string, data map[string]interface{}) error { if nil != p { _, err := p.cli.Update(). Index(index). @@ -126,7 +124,8 @@ func (p *ElkEngine)Update(index string,types string,id string,data map[string]in } return errors.New(ERROR_PTR) } -func (p *ElkEngine)CreateIndex(index string,typemaping string) error{ + +func (p *ElkEngine) CreateIndex(index string, typemaping string) error { if nil != p { exists, err := p.cli.IndexExists(index).Do(context.Background()) if err != nil { @@ -134,7 +133,7 @@ func (p *ElkEngine)CreateIndex(index string,typemaping string) error{ log.Print(err) return err } - if exists{ + if exists { return errors.New(INDEX_EXISTED) } createIndex, err := p.cli.CreateIndex(index).Body(typemaping).Do(context.Background()) @@ -146,24 +145,23 @@ func (p *ElkEngine)CreateIndex(index string,typemaping string) error{ return errors.New("create index error") // Not acknowledged } - return nil + return nil } return errors.New(ERROR_PTR) } -func (p *ElkEngine)IndexExisted(index string) (bool,error ){ +func (p *ElkEngine) IndexExisted(index string) (bool, error) { if nil != p { exists, err := p.cli.IndexExists(index).Do(context.Background()) - if exists{ - return true,nil + if exists { + return true, nil } if err != nil { // Handle error log.Print(err) - return false,err + return false, err } - return false,nil - + return false, nil } - return false,nil -} \ No newline at end of file + return false, nil +} diff --git a/test/portData_test.go b/test/portData_test.go index 4557260..cde8562 100644 --- a/test/portData_test.go +++ b/test/portData_test.go @@ -55,22 +55,21 @@ func TestPortDocToElastic(t *testing.T) { } } - -func TestReflect(t *testing.T){ - type XX struct{ - A int16 `json:"bb"` +func TestReflect(t *testing.T) { + type XX struct { + A int16 `json:"bb"` B string `json: "cc" xml:"test"` } - unmar := map[string] interface{}{ - "bb" : 2, + unmar := map[string]interface{}{ + "bb": 2, "cc": "hello", } xx := XX{} - utils.UnmarshalJson(&xx,unmar) + utils.UnmarshalJson2Struct(&xx, unmar) log.Print(xx) } -func TestQueryDoc(t *testing.T){ +func TestQueryDoc(t *testing.T) { InitConfig() InitLogs() InitRedisConfig() @@ -78,11 +77,18 @@ func TestQueryDoc(t *testing.T){ db.InitELK() query := elastic.NewTermQuery("content", "title") - //_ = elastic.NewQueryStringQuery(fieldValue) //关键字查询 - searchResult,titles,e := db.GetElastic().Query("doc",query,reflect.TypeOf(model.Doc{}),10,0) - if nil != e{ + searchResult, titles, e := db.GetElastic().Query("doc", query, reflect.TypeOf(model.Doc{}), 10, 0) + if nil != e { log.Print(e.Error()) } log.Print(searchResult) log.Print(titles) -} \ No newline at end of file +} + +func TestChangeStructFieldThroughStruct(t *testing.T) { + type XX struct { + A int16 `json:"bb"` + B string `json: "cc" xml:"test"` + } + +} diff --git a/utils/reflect.go b/utils/reflect.go index 847e884..d7f7d2d 100644 --- a/utils/reflect.go +++ b/utils/reflect.go @@ -4,153 +4,162 @@ import ( "log" "reflect" "strings" + + "github.com/pkg/errors" ) +const ( + ErrorTypeError = "error input interface type error" + TagNoExisted = "error remap tag no existed" +) -func ReflectMakeNew(t reflect.Type) interface{} { +func reflectMakeNew(t reflect.Type) interface{} { retptr := reflect.New(t) sval := retptr.Elem().Interface() return sval } -type TagMap map[string] string +type TagMap map[string]string -func ReflectTagMap(t reflect.Type) map[string] TagMap{ - log.Print(t.Name()) - ret := make(map[string] TagMap) +func ReflectTagMap(t reflect.Type) map[string]TagMap { + // log.Print(t.Kind(), " and ", t.Name()) + if t.Kind() != reflect.Struct { + return nil + } + ret := make(map[string]TagMap) num := t.NumField() - for i := 0;i < num;i++{ - sub := strings.Split(string(t.Field(i).Tag),"\"") - for k,v := range(sub){ - if len(v) > 0{ - if k%2 == 0{ - v = strings.Trim(v," ") + for i := 0; i < num; i++ { + sub := strings.Split(string(t.Field(i).Tag), "\"") + for k, v := range sub { + if len(v) > 0 { + if k%2 == 0 { + v = strings.Trim(v, " ") } - if v[len(v) - 1] == ':'{ - if _,ok := ret[v[:len(v) - 1]];ok{ - if ((k + 1) < len(sub) - 1) && ((sub[k + 1][len(sub[k + 1]) - 1] ) == ':'){ - continue; - }else{ - ret[v[:len(v) - 1]][sub[k + 1]] = string(t.Field(i).Name) + if v[len(v)-1] == ':' { + if _, ok := ret[v[:len(v)-1]]; ok { + if ((k + 1) < len(sub)-1) && ((sub[k+1][len(sub[k+1])-1]) == ':') { + continue + } else { + ret[v[:len(v)-1]][sub[k+1]] = string(t.Field(i).Name) } - }else{ - ret[v[:len(v) - 1]] = make(TagMap) + } else { + ret[v[:len(v)-1]] = make(TagMap) log.Print() - if ((k + 1) < len(sub) - 1) && ((sub[k + 1][len(sub[k + 1]) - 1] ) == ':'){ - continue; - }else{ - ret[v[:len(v) - 1]][sub[k + 1]] = string(t.Field(i).Name) + if ((k + 1) < len(sub)-1) && ((sub[k+1][len(sub[k+1])-1]) == ':') { + continue + } else { + ret[v[:len(v)-1]][sub[k+1]] = string(t.Field(i).Name) } } - }else{ + } else { continue } } - + } } return ret } -func SameKind(typ1 reflect.Kind,typ2 reflect.Kind) bool{ - switch typ1{ +func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool { + switch typ1 { case reflect.Int: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } case reflect.Int8: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true - } + } case reflect.Int16: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } case reflect.Int32: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true - } + } case reflect.Uint: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } case reflect.Uint8: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } case reflect.Uint16: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true - } + } case reflect.Uint32: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } case reflect.Uint64: - if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true } - if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true - } + } case reflect.Float32: - if(typ2 == reflect.Float32 ||typ2 == reflect.Float64){ + if typ2 == reflect.Float32 || typ2 == reflect.Float64 { return true } case reflect.Float64: - if(typ2 == reflect.Float32 ||typ2 == reflect.Float64){ + if typ2 == reflect.Float32 || typ2 == reflect.Float64 { return true } default: - return (typ1 == typ2) + return (typ1 == typ2) } return false } -func UnmarshalWithTag(value interface{}, maps map[string]interface{},tag string){ - if "" == tag{ +func UnmarshalWithTag(value interface{}, maps map[string]interface{}, tag string) { + if "" == tag { return } valueof := reflect.ValueOf(value) - if !valueof.Elem().CanAddr(){ + if !valueof.Elem().CanAddr() { log.Print("should be addr") return } remap := ReflectTagMap(valueof.Elem().Type()) - _,ok := remap[tag] - if !ok{ - return + _, ok := remap[tag] + if !ok { + return } - for k,v := range(maps){ + for k, v := range maps { log.Print(k) - if filedName,ok := remap["json"][k];ok{ - log.Print(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind().String()) - if SameKind(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind()){ - if(valueof.Elem().FieldByName(filedName).CanSet()){ + if filedName, ok := remap["json"][k]; ok { + log.Print(valueof.Elem().FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String()) + if SameKind(valueof.Elem().FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) { + if valueof.Elem().FieldByName(filedName).CanSet() { valueof.Elem().FieldByName(filedName).Set(reflect.ValueOf(v).Convert(valueof.Elem().FieldByName(filedName).Type())) } } @@ -158,29 +167,81 @@ func UnmarshalWithTag(value interface{}, maps map[string]interface{},tag string) } } - - -func UnmarshalJson(value interface{}, maps map[string]interface{}){ +/* + an generaly unmarshal function implement + accept only pointer to struct or pointer to interface contains struct +*/ +func UnmarshalJson2Struct(value interface{}, maps map[string]interface{}) error { + log.Print("UnmarshalJson kind is ", reflect.ValueOf(value).Kind()) + log.Print("UnmarshalJson kind1 is ", reflect.ValueOf(value).Elem().Kind()) + // log.Print("UnmarshalJson kind2 is ", reflect.ValueOf(value).Elem().Elem().Kind()) valueof := reflect.ValueOf(value) - if !valueof.Elem().CanAddr(){ - log.Print("should be addr") - return + var opStruct reflect.Value + if valueof.Kind() == reflect.Ptr { + // if it is ptr to interface,the interface must contains a struct type + if valueof.Elem().Kind() == reflect.Interface { + if valueof.Elem().Elem().Kind() == reflect.Struct { + opStruct = valueof.Elem().Elem() + } else { + return errors.New(ErrorTypeError) + } + } else if valueof.Elem().Kind() == reflect.Struct { + // simply it is ptr to struct + opStruct = valueof.Elem() + } else { + return errors.New(ErrorTypeError) + } + } else { + return errors.New(ErrorTypeError) } - remap := ReflectTagMap(valueof.Elem().Type()) - _,ok := remap["json"] - if !ok{ - return + + remap := ReflectTagMap(opStruct.Type()) + _, ok := remap["json"] + if !ok { + return errors.New(TagNoExisted) } - for k,v := range(maps){ + for k, v := range maps { log.Print(k) - if filedName,ok := remap["json"][k];ok{ - log.Print(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind().String()) - if SameKind(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind()){ - if(valueof.Elem().FieldByName(filedName).CanSet()){ - valueof.Elem().FieldByName(filedName).Set(reflect.ValueOf(v).Convert(valueof.Elem().FieldByName(filedName).Type())) + if filedName, ok := remap["json"][k]; ok { + log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String()) + if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) { + if opStruct.FieldByName(filedName).CanSet() { + opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type())) } } } } + return nil } +/* + an generaly unmarshal function implement + accept only pointer to struct or pointer to interface contains struct +*/ +func UnmarshalJson2StructGen(t reflect.Type, maps map[string]interface{}) (interface{}, error) { + if t.Kind() != reflect.Struct { + return nil, errors.New(ErrorTypeError) + } + remap := ReflectTagMap(t) + _, ok := remap["json"] + if !ok { + return nil, errors.New(TagNoExisted) + } + ret := reflect.New(t) + // log.Print(ret.Kind()) + opStruct := ret.Elem() + // log.Print(opStruct.Kind()) + for k, v := range maps { + log.Print(k) + if filedName, ok := remap["json"][k]; ok { + // log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String()) + if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) { + // log.Print(opStruct.FieldByName(filedName).CanAddr(), opStruct.FieldByName(filedName).CanSet()) + if opStruct.FieldByName(filedName).CanSet() { + opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type())) + } + } + } + } + return ret.Elem().Interface(), nil +}