完善es engine

master
zcy 2021-02-07 13:14:01 +08:00
parent 3a49c9f4c9
commit 92c76111b8
3 changed files with 214 additions and 149 deletions

View File

@ -10,109 +10,107 @@ import (
"gopkg.in/olivere/elastic.v7"
"qiniupkg.com/x/log.v7"
)
const(
ERROR_PTR = "null pointer error"
INPUT_TYPE_ERROR = "wrong input parameter"
CREATED_ERROR = "create error"
DELETE_ERROR = "delete error"
INDEX_EXISTED = "index existed"
const (
ERROR_PTR = "null pointer error"
INPUT_TYPE_ERROR = "wrong input parameter"
CREATED_ERROR = "create error"
DELETE_ERROR = "delete error"
INDEX_EXISTED = "index existed"
)
type ElkEngine struct {
type ElkEngine struct {
cli *elastic.Client
}
func (p *ElkEngine)Create(index string,types string,id string,data interface{}) (error) {
if nil != p{
if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct){
func (p *ElkEngine) Create(index string, types string, id string, data interface{}) error {
if nil != p {
if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct) {
resp, err := p.cli.Index().
Index(index).
BodyJson(data).
Do(context.Background())
if err != nil {
log.Print("create error",err)
log.Print("create error", err)
return err
}
log.Print(resp)
}else{
} else {
log.Print(reflect.TypeOf(data).Kind())
return errors.New(INPUT_TYPE_ERROR)
}
}else{
} else {
return errors.New(ERROR_PTR)
}
return nil
}
func (p *ElkEngine)Delete(query elastic.Query,index string) error{
if nil != p{
func (p *ElkEngine) Delete(query elastic.Query, index string) error {
if nil != p {
_, err := p.cli.DeleteByQuery().Index(index).Query(query).
Do(context.Background())
if err != nil {
log.Print(err)
return err
return err
}
}else{
} else {
return errors.New(ERROR_PTR)
}
return nil
}
/*
*/
func (p *ElkEngine)Query(index string,query elastic.Query,typ reflect.Type,
limit int,offset int) ([]interface{},[]string,error) {
func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type,
limit int, offset int) ([]interface{}, []string, error) {
rets := []interface{}{}
if nil != p{
if(limit == 0){
if nil != p {
if limit == 0 {
res, err := p.cli.Search(index).Query(query).Do(context.Background())
if err != nil {
print(err)
return nil,nil,err
return nil, nil, err
}
id := []string{}
for _,vs := range res.Hits.Hits{
id = append(id,vs.Id)
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
}
return rets,id,nil
}else{
res, err := p.cli.Search(index).Query(query).Size(limit).From(limit*offset).Do(context.Background())
return rets, id, nil
} else {
res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background())
if err != nil {
print(err)
return nil,nil,err
return nil, nil, err
}
id := []string{}
for _,vs := range res.Hits.Hits{
id = append(id,vs.Id)
data,e := vs.Source.MarshalJSON()
if nil != e{
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
data, e := vs.Source.MarshalJSON()
if nil != e {
log.Print(e.Error())
}
obj := utils.ReflectMakeNew(typ)
mapobj := map[string]interface{}{}
e = json.Unmarshal(data,&mapobj)
if nil != e{
e = json.Unmarshal(data, &mapobj)
if nil != e {
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(typ, mapobj)
if nil != e {
log.Print(e.Error())
}
// for k,_ := range mapobj{
// value := reflect.ValueOf(obj)
// }
log.Print(obj)
rets = append(rets,obj)
log.Print(string(data))
rets = append(rets, obj)
}
return rets,id,nil
return rets, id, nil
}
}else{
return nil,nil,errors.New(ERROR_PTR)
} else {
return nil, nil, errors.New(ERROR_PTR)
}
}
func (p *ElkEngine)Update(index string,types string,id string,data map[string]interface{}) error {
func (p *ElkEngine) Update(index string, types string, id string, data map[string]interface{}) error {
if nil != p {
_, err := p.cli.Update().
Index(index).
@ -126,7 +124,8 @@ func (p *ElkEngine)Update(index string,types string,id string,data map[string]in
}
return errors.New(ERROR_PTR)
}
func (p *ElkEngine)CreateIndex(index string,typemaping string) error{
func (p *ElkEngine) CreateIndex(index string, typemaping string) error {
if nil != p {
exists, err := p.cli.IndexExists(index).Do(context.Background())
if err != nil {
@ -134,7 +133,7 @@ func (p *ElkEngine)CreateIndex(index string,typemaping string) error{
log.Print(err)
return err
}
if exists{
if exists {
return errors.New(INDEX_EXISTED)
}
createIndex, err := p.cli.CreateIndex(index).Body(typemaping).Do(context.Background())
@ -146,24 +145,23 @@ func (p *ElkEngine)CreateIndex(index string,typemaping string) error{
return errors.New("create index error")
// Not acknowledged
}
return nil
return nil
}
return errors.New(ERROR_PTR)
}
func (p *ElkEngine)IndexExisted(index string) (bool,error ){
func (p *ElkEngine) IndexExisted(index string) (bool, error) {
if nil != p {
exists, err := p.cli.IndexExists(index).Do(context.Background())
if exists{
return true,nil
if exists {
return true, nil
}
if err != nil {
// Handle error
log.Print(err)
return false,err
return false, err
}
return false,nil
return false, nil
}
return false,nil
return false, nil
}

View File

@ -55,22 +55,21 @@ func TestPortDocToElastic(t *testing.T) {
}
}
func TestReflect(t *testing.T){
type XX struct{
A int16 `json:"bb"`
func TestReflect(t *testing.T) {
type XX struct {
A int16 `json:"bb"`
B string `json: "cc" xml:"test"`
}
unmar := map[string] interface{}{
"bb" : 2,
unmar := map[string]interface{}{
"bb": 2,
"cc": "hello",
}
xx := XX{}
utils.UnmarshalJson(&xx,unmar)
utils.UnmarshalJson2Struct(&xx, unmar)
log.Print(xx)
}
func TestQueryDoc(t *testing.T){
func TestQueryDoc(t *testing.T) {
InitConfig()
InitLogs()
InitRedisConfig()
@ -78,11 +77,18 @@ func TestQueryDoc(t *testing.T){
db.InitELK()
query := elastic.NewTermQuery("content", "title")
//_ = elastic.NewQueryStringQuery(fieldValue) //关键字查询
searchResult,titles,e := db.GetElastic().Query("doc",query,reflect.TypeOf(model.Doc{}),10,0)
if nil != e{
searchResult, titles, e := db.GetElastic().Query("doc", query, reflect.TypeOf(model.Doc{}), 10, 0)
if nil != e {
log.Print(e.Error())
}
log.Print(searchResult)
log.Print(titles)
}
func TestChangeStructFieldThroughStruct(t *testing.T) {
type XX struct {
A int16 `json:"bb"`
B string `json: "cc" xml:"test"`
}
}

View File

@ -4,45 +4,54 @@ import (
"log"
"reflect"
"strings"
"github.com/pkg/errors"
)
const (
ErrorTypeError = "error input interface type error"
TagNoExisted = "error remap tag no existed"
)
func ReflectMakeNew(t reflect.Type) interface{} {
func reflectMakeNew(t reflect.Type) interface{} {
retptr := reflect.New(t)
sval := retptr.Elem().Interface()
return sval
}
type TagMap map[string] string
type TagMap map[string]string
func ReflectTagMap(t reflect.Type) map[string] TagMap{
log.Print(t.Name())
ret := make(map[string] TagMap)
func ReflectTagMap(t reflect.Type) map[string]TagMap {
// log.Print(t.Kind(), " and ", t.Name())
if t.Kind() != reflect.Struct {
return nil
}
ret := make(map[string]TagMap)
num := t.NumField()
for i := 0;i < num;i++{
sub := strings.Split(string(t.Field(i).Tag),"\"")
for k,v := range(sub){
if len(v) > 0{
if k%2 == 0{
v = strings.Trim(v," ")
for i := 0; i < num; i++ {
sub := strings.Split(string(t.Field(i).Tag), "\"")
for k, v := range sub {
if len(v) > 0 {
if k%2 == 0 {
v = strings.Trim(v, " ")
}
if v[len(v) - 1] == ':'{
if _,ok := ret[v[:len(v) - 1]];ok{
if ((k + 1) < len(sub) - 1) && ((sub[k + 1][len(sub[k + 1]) - 1] ) == ':'){
continue;
}else{
ret[v[:len(v) - 1]][sub[k + 1]] = string(t.Field(i).Name)
if v[len(v)-1] == ':' {
if _, ok := ret[v[:len(v)-1]]; ok {
if ((k + 1) < len(sub)-1) && ((sub[k+1][len(sub[k+1])-1]) == ':') {
continue
} else {
ret[v[:len(v)-1]][sub[k+1]] = string(t.Field(i).Name)
}
}else{
ret[v[:len(v) - 1]] = make(TagMap)
} else {
ret[v[:len(v)-1]] = make(TagMap)
log.Print()
if ((k + 1) < len(sub) - 1) && ((sub[k + 1][len(sub[k + 1]) - 1] ) == ':'){
continue;
}else{
ret[v[:len(v) - 1]][sub[k + 1]] = string(t.Field(i).Name)
if ((k + 1) < len(sub)-1) && ((sub[k+1][len(sub[k+1])-1]) == ':') {
continue
} else {
ret[v[:len(v)-1]][sub[k+1]] = string(t.Field(i).Name)
}
}
}else{
} else {
continue
}
}
@ -52,77 +61,77 @@ func ReflectTagMap(t reflect.Type) map[string] TagMap{
return ret
}
func SameKind(typ1 reflect.Kind,typ2 reflect.Kind) bool{
switch typ1{
func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
switch typ1 {
case reflect.Int:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Int8:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Int16:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Int32:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Uint:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Uint8:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Uint16:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Uint32:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Uint64:
if(typ2 == reflect.Int ||typ2 == reflect.Int16 ||typ2 == reflect.Int32 ||typ2 == reflect.Int64||typ2 == reflect.Int8){
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if(typ2 == reflect.Uint ||typ2 == reflect.Uint16 ||typ2 == reflect.Uint32 ||typ2 == reflect.Uint64||typ2 == reflect.Uint8){
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
case reflect.Float32:
if(typ2 == reflect.Float32 ||typ2 == reflect.Float64){
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Float64:
if(typ2 == reflect.Float32 ||typ2 == reflect.Float64){
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
default:
@ -131,26 +140,26 @@ func SameKind(typ1 reflect.Kind,typ2 reflect.Kind) bool{
return false
}
func UnmarshalWithTag(value interface{}, maps map[string]interface{},tag string){
if "" == tag{
func UnmarshalWithTag(value interface{}, maps map[string]interface{}, tag string) {
if "" == tag {
return
}
valueof := reflect.ValueOf(value)
if !valueof.Elem().CanAddr(){
if !valueof.Elem().CanAddr() {
log.Print("should be addr")
return
}
remap := ReflectTagMap(valueof.Elem().Type())
_,ok := remap[tag]
if !ok{
_, ok := remap[tag]
if !ok {
return
}
for k,v := range(maps){
for k, v := range maps {
log.Print(k)
if filedName,ok := remap["json"][k];ok{
log.Print(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind().String())
if SameKind(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind()){
if(valueof.Elem().FieldByName(filedName).CanSet()){
if filedName, ok := remap["json"][k]; ok {
log.Print(valueof.Elem().FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
if SameKind(valueof.Elem().FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) {
if valueof.Elem().FieldByName(filedName).CanSet() {
valueof.Elem().FieldByName(filedName).Set(reflect.ValueOf(v).Convert(valueof.Elem().FieldByName(filedName).Type()))
}
}
@ -158,29 +167,81 @@ func UnmarshalWithTag(value interface{}, maps map[string]interface{},tag string)
}
}
func UnmarshalJson(value interface{}, maps map[string]interface{}){
/*
an generaly unmarshal function implement
accept only pointer to struct or pointer to interface contains struct
*/
func UnmarshalJson2Struct(value interface{}, maps map[string]interface{}) error {
log.Print("UnmarshalJson kind is ", reflect.ValueOf(value).Kind())
log.Print("UnmarshalJson kind1 is ", reflect.ValueOf(value).Elem().Kind())
// log.Print("UnmarshalJson kind2 is ", reflect.ValueOf(value).Elem().Elem().Kind())
valueof := reflect.ValueOf(value)
if !valueof.Elem().CanAddr(){
log.Print("should be addr")
return
var opStruct reflect.Value
if valueof.Kind() == reflect.Ptr {
// if it is ptr to interface,the interface must contains a struct type
if valueof.Elem().Kind() == reflect.Interface {
if valueof.Elem().Elem().Kind() == reflect.Struct {
opStruct = valueof.Elem().Elem()
} else {
return errors.New(ErrorTypeError)
}
} else if valueof.Elem().Kind() == reflect.Struct {
// simply it is ptr to struct
opStruct = valueof.Elem()
} else {
return errors.New(ErrorTypeError)
}
} else {
return errors.New(ErrorTypeError)
}
remap := ReflectTagMap(valueof.Elem().Type())
_,ok := remap["json"]
if !ok{
return
remap := ReflectTagMap(opStruct.Type())
_, ok := remap["json"]
if !ok {
return errors.New(TagNoExisted)
}
for k,v := range(maps){
for k, v := range maps {
log.Print(k)
if filedName,ok := remap["json"][k];ok{
log.Print(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind().String())
if SameKind(valueof.Elem().FieldByName(filedName).Kind(),reflect.TypeOf(v).Kind()){
if(valueof.Elem().FieldByName(filedName).CanSet()){
valueof.Elem().FieldByName(filedName).Set(reflect.ValueOf(v).Convert(valueof.Elem().FieldByName(filedName).Type()))
if filedName, ok := remap["json"][k]; ok {
log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) {
if opStruct.FieldByName(filedName).CanSet() {
opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type()))
}
}
}
}
return nil
}
/*
an generaly unmarshal function implement
accept only pointer to struct or pointer to interface contains struct
*/
func UnmarshalJson2StructGen(t reflect.Type, maps map[string]interface{}) (interface{}, error) {
if t.Kind() != reflect.Struct {
return nil, errors.New(ErrorTypeError)
}
remap := ReflectTagMap(t)
_, ok := remap["json"]
if !ok {
return nil, errors.New(TagNoExisted)
}
ret := reflect.New(t)
// log.Print(ret.Kind())
opStruct := ret.Elem()
// log.Print(opStruct.Kind())
for k, v := range maps {
log.Print(k)
if filedName, ok := remap["json"][k]; ok {
// log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) {
// log.Print(opStruct.FieldByName(filedName).CanAddr(), opStruct.FieldByName(filedName).CanSet())
if opStruct.FieldByName(filedName).CanSet() {
opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type()))
}
}
}
}
return ret.Elem().Interface(), nil
}