caiyu/background
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完善es engine

Browse Source
This commit is contained in:
zcy 2021-02-07 13:14:01 +08:00
parent 3a49c9f4c9
commit 92c76111b8
3 changed files with 214 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
db/elasticEngine.go
View File

@ -10,21 +10,20 @@ import (
"gopkg.in/olivere/elastic.v7" "gopkg.in/olivere/elastic.v7"
"qiniupkg.com/x/log.v7" "qiniupkg.com/x/log.v7"
) )
const ( const (
ERROR_PTR = "null pointer error" ERROR_PTR = "null pointer error"
INPUT_TYPE_ERROR = "wrong input parameter" INPUT_TYPE_ERROR = "wrong input parameter"
CREATED_ERROR = "create error" CREATED_ERROR = "create error"
DELETE_ERROR = "delete error" DELETE_ERROR = "delete error"
INDEX_EXISTED = "index existed" INDEX_EXISTED = "index existed"
) )
type ElkEngine struct { type ElkEngine struct {
cli *elastic.Client cli *elastic.Client
} }
func (p *ElkEngine) Create(index string, types string, id string, data interface{}) error {
func (p *ElkEngine)Create(index string,types string,id string,data interface{}) (error) {
if nil != p { if nil != p {
if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct) { if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct) {
resp, err := p.cli.Index(). resp, err := p.cli.Index().
@ -60,14 +59,12 @@ func (p *ElkEngine)Delete(query elastic.Query,index string) error{
} }
return nil return nil
} }
/*
*/
func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type, func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type,
limit int, offset int) ([]interface{}, []string, error) { limit int, offset int) ([]interface{}, []string, error) {
rets := []interface{}{} rets := []interface{}{}
if nil != p { if nil != p {
if(limit == 0){ if limit == 0 {
res, err := p.cli.Search(index).Query(query).Do(context.Background()) res, err := p.cli.Search(index).Query(query).Do(context.Background())
if err != nil { if err != nil {
print(err) print(err)
@ -92,18 +89,19 @@ func (p *ElkEngine)Query(index string,query elastic.Query,typ reflect.Type,
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
obj := utils.ReflectMakeNew(typ)
mapobj := map[string]interface{}{} mapobj := map[string]interface{}{}
e = json.Unmarshal(data, &mapobj) e = json.Unmarshal(data, &mapobj)
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
obj, e := utils.UnmarshalJson2StructGen(typ, mapobj)
if nil != e {
log.Print(e.Error())
}
// for k,_ := range mapobj{ // for k,_ := range mapobj{
// value := reflect.ValueOf(obj) // value := reflect.ValueOf(obj)
// } // }
log.Print(obj)
rets = append(rets, obj) rets = append(rets, obj)
log.Print(string(data))
} }
return rets, id, nil return rets, id, nil
} }
@ -126,6 +124,7 @@ func (p *ElkEngine)Update(index string,types string,id string,data map[string]in
} }
return errors.New(ERROR_PTR) return errors.New(ERROR_PTR)
} }
func (p *ElkEngine) CreateIndex(index string, typemaping string) error { func (p *ElkEngine) CreateIndex(index string, typemaping string) error {
if nil != p { if nil != p {
exists, err := p.cli.IndexExists(index).Do(context.Background()) exists, err := p.cli.IndexExists(index).Do(context.Background())
@ -163,7 +162,6 @@ func (p *ElkEngine)IndexExisted(index string) (bool,error ){
return false, err return false, err
} }
return false, nil return false, nil
} }
return false, nil return false, nil
} }

12
test/portData_test.go
View File

@ -55,7 +55,6 @@ func TestPortDocToElastic(t *testing.T) {
} }
} }
func TestReflect(t *testing.T) { func TestReflect(t *testing.T) {
type XX struct { type XX struct {
A int16 `json:"bb"` A int16 `json:"bb"`
@ -66,7 +65,7 @@ func TestReflect(t *testing.T){
"cc": "hello", "cc": "hello",
} }
xx := XX{} xx := XX{}
utils.UnmarshalJson(&xx,unmar) utils.UnmarshalJson2Struct(&xx, unmar)
log.Print(xx) log.Print(xx)
} }
@ -78,7 +77,6 @@ func TestQueryDoc(t *testing.T){
db.InitELK() db.InitELK()
query := elastic.NewTermQuery("content", "title") query := elastic.NewTermQuery("content", "title")
//_ = elastic.NewQueryStringQuery(fieldValue) //关键字查询
searchResult, titles, e := db.GetElastic().Query("doc", query, reflect.TypeOf(model.Doc{}), 10, 0) searchResult, titles, e := db.GetElastic().Query("doc", query, reflect.TypeOf(model.Doc{}), 10, 0)
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
@ -86,3 +84,11 @@ func TestQueryDoc(t *testing.T){
log.Print(searchResult) log.Print(searchResult)
log.Print(titles) log.Print(titles)
} }
func TestChangeStructFieldThroughStruct(t *testing.T) {
type XX struct {
A int16 `json:"bb"`
B string `json: "cc" xml:"test"`
}
}

141
utils/reflect.go
View File

@ -4,10 +4,16 @@ import (
"log" "log"
"reflect" "reflect"
"strings" "strings"
"github.com/pkg/errors"
) )
const (
ErrorTypeError = "error input interface type error"
TagNoExisted = "error remap tag no existed"
)
func ReflectMakeNew(t reflect.Type) interface{} { func reflectMakeNew(t reflect.Type) interface{} {
retptr := reflect.New(t) retptr := reflect.New(t)
sval := retptr.Elem().Interface() sval := retptr.Elem().Interface()
return sval return sval
@ -16,12 +22,15 @@ func ReflectMakeNew(t reflect.Type) interface{} {
type TagMap map[string]string type TagMap map[string]string
func ReflectTagMap(t reflect.Type) map[string]TagMap { func ReflectTagMap(t reflect.Type) map[string]TagMap {
log.Print(t.Name()) // log.Print(t.Kind(), " and ", t.Name())
if t.Kind() != reflect.Struct {
return nil
}
ret := make(map[string]TagMap) ret := make(map[string]TagMap)
num := t.NumField() num := t.NumField()
for i := 0; i < num; i++ { for i := 0; i < num; i++ {
sub := strings.Split(string(t.Field(i).Tag), "\"") sub := strings.Split(string(t.Field(i).Tag), "\"")
for k,v := range(sub){ for k, v := range sub {
if len(v) > 0 { if len(v) > 0 {
if k%2 == 0 { if k%2 == 0 {
v = strings.Trim(v, " ") v = strings.Trim(v, " ")
@ -29,7 +38,7 @@ func ReflectTagMap(t reflect.Type) map[string] TagMap{
if v[len(v)-1] == ':' { if v[len(v)-1] == ':' {
if _, ok := ret[v[:len(v)-1]]; ok { if _, ok := ret[v[:len(v)-1]]; ok {
if ((k + 1) < len(sub)-1) && ((sub[k+1][len(sub[k+1])-1]) == ':') { if ((k + 1) < len(sub)-1) && ((sub[k+1][len(sub[k+1])-1]) == ':') {
continue; continue
} else { } else {
ret[v[:len(v)-1]][sub[k+1]] = string(t.Field(i).Name) ret[v[:len(v)-1]][sub[k+1]] = string(t.Field(i).Name)
} }
@ -37,7 +46,7 @@ func ReflectTagMap(t reflect.Type) map[string] TagMap{
ret[v[:len(v)-1]] = make(TagMap) ret[v[:len(v)-1]] = make(TagMap)
log.Print() log.Print()
if ((k + 1) < len(sub)-1) && ((sub[k+1][len(sub[k+1])-1]) == ':') { if ((k + 1) < len(sub)-1) && ((sub[k+1][len(sub[k+1])-1]) == ':') {
continue; continue
} else { } else {
ret[v[:len(v)-1]][sub[k+1]] = string(t.Field(i).Name) ret[v[:len(v)-1]][sub[k+1]] = string(t.Field(i).Name)
} }
@ -55,74 +64,74 @@ func ReflectTagMap(t reflect.Type) map[string] TagMap{
func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool { func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
switch typ1 { switch typ1 {
case reflect.Int: case reflect.Int:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Int8: case reflect.Int8:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Int16: case reflect.Int16:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Int32: case reflect.Int32:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Uint: case reflect.Uint:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Uint8: case reflect.Uint8:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Uint16: case reflect.Uint16:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Uint32: case reflect.Uint32:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Uint64: case reflect.Uint64:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){ if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true return true
} }
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){ if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true return true
} }
case reflect.Float32: case reflect.Float32:
if(typ2 == reflect.Float32 ||typ2 == reflect.Float64){ if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true return true
} }
case reflect.Float64: case reflect.Float64:
if(typ2 == reflect.Float32 ||typ2 == reflect.Float64){ if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true return true
} }
default: default:
@ -145,12 +154,12 @@ func UnmarshalWithTag(value interface{}, maps map[string]interface{},tag string)
if !ok { if !ok {
return return
} }
for k,v := range(maps){ for k, v := range maps {
log.Print(k) log.Print(k)
if filedName, ok := remap["json"][k]; ok { if filedName, ok := remap["json"][k]; ok {
log.Print(valueof.Elem().FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String()) log.Print(valueof.Elem().FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
if SameKind(valueof.Elem().FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) { if SameKind(valueof.Elem().FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) {
if(valueof.Elem().FieldByName(filedName).CanSet()){ if valueof.Elem().FieldByName(filedName).CanSet() {
valueof.Elem().FieldByName(filedName).Set(reflect.ValueOf(v).Convert(valueof.Elem().FieldByName(filedName).Type())) valueof.Elem().FieldByName(filedName).Set(reflect.ValueOf(v).Convert(valueof.Elem().FieldByName(filedName).Type()))
} }
} }
@ -158,29 +167,81 @@ func UnmarshalWithTag(value interface{}, maps map[string]interface{},tag string)
} }
} }
/*
an generaly unmarshal function implement
func UnmarshalJson(value interface{}, maps map[string]interface{}){ accept only pointer to struct or pointer to interface contains struct
*/
func UnmarshalJson2Struct(value interface{}, maps map[string]interface{}) error {
log.Print("UnmarshalJson kind is ", reflect.ValueOf(value).Kind())
log.Print("UnmarshalJson kind1 is ", reflect.ValueOf(value).Elem().Kind())
// log.Print("UnmarshalJson kind2 is ", reflect.ValueOf(value).Elem().Elem().Kind())
valueof := reflect.ValueOf(value) valueof := reflect.ValueOf(value)
if !valueof.Elem().CanAddr(){ var opStruct reflect.Value
log.Print("should be addr") if valueof.Kind() == reflect.Ptr {
return // if it is ptr to interface,the interface must contains a struct type
if valueof.Elem().Kind() == reflect.Interface {
if valueof.Elem().Elem().Kind() == reflect.Struct {
opStruct = valueof.Elem().Elem()
} else {
return errors.New(ErrorTypeError)
} }
remap := ReflectTagMap(valueof.Elem().Type()) } else if valueof.Elem().Kind() == reflect.Struct {
// simply it is ptr to struct
opStruct = valueof.Elem()
} else {
return errors.New(ErrorTypeError)
}
} else {
return errors.New(ErrorTypeError)
}
remap := ReflectTagMap(opStruct.Type())
_, ok := remap["json"] _, ok := remap["json"]
if !ok { if !ok {
return return errors.New(TagNoExisted)
} }
for k,v := range(maps){ for k, v := range maps {
log.Print(k) log.Print(k)
if filedName, ok := remap["json"][k]; ok { if filedName, ok := remap["json"][k]; ok {
log.Print(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind().String()) log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
if SameKind(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind()){ if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) {
if(valueof.Elem().FieldByName(filedName).CanSet()){ if opStruct.FieldByName(filedName).CanSet() {
valueof.Elem().FieldByName(filedName).Set(reflect.ValueOf(v).Convert(valueof.Elem().FieldByName(filedName).Type())) opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type()))
} }
} }
} }
} }
return nil
} }
/*
an generaly unmarshal function implement
accept only pointer to struct or pointer to interface contains struct
*/
func UnmarshalJson2StructGen(t reflect.Type, maps map[string]interface{}) (interface{}, error) {
if t.Kind() != reflect.Struct {
return nil, errors.New(ErrorTypeError)
}
remap := ReflectTagMap(t)
_, ok := remap["json"]
if !ok {
return nil, errors.New(TagNoExisted)
}
ret := reflect.New(t)
// log.Print(ret.Kind())
opStruct := ret.Elem()
// log.Print(opStruct.Kind())
for k, v := range maps {
log.Print(k)
if filedName, ok := remap["json"][k]; ok {
// log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) {
// log.Print(opStruct.FieldByName(filedName).CanAddr(), opStruct.FieldByName(filedName).CanSet())
if opStruct.FieldByName(filedName).CanSet() {
opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type()))
}
}
}
}
return ret.Elem().Interface(), nil
}