background/db/elasticEngine.go

321 lines
7.8 KiB
Go
Raw Normal View History

2020-03-13 09:40:00 +00:00
package db
import (
2021-02-05 17:46:59 +00:00
"background/utils"
"encoding/json"
"reflect"
2020-03-20 17:50:04 +00:00
"github.com/pkg/errors"
"golang.org/x/net/context"
"gopkg.in/olivere/elastic.v7"
"qiniupkg.com/x/log.v7"
2020-03-13 09:40:00 +00:00
)
2021-02-07 05:14:01 +00:00
const (
ERROR_PTR = "null pointer error"
2021-02-07 09:21:19 +00:00
INPUT_TYPE_ERROR = "wrong input parameter: "
2021-02-07 05:14:01 +00:00
CREATED_ERROR = "create error"
DELETE_ERROR = "delete error"
INDEX_EXISTED = "index existed"
2020-03-20 17:50:04 +00:00
)
2021-02-07 05:14:01 +00:00
type ElkEngine struct {
2020-03-13 09:40:00 +00:00
cli *elastic.Client
}
2021-02-07 05:14:01 +00:00
func (p *ElkEngine) Create(index string, types string, id string, data interface{}) error {
if nil != p {
if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct) {
2020-03-20 17:50:04 +00:00
resp, err := p.cli.Index().
Index(index).
BodyJson(data).
Do(context.Background())
2020-03-20 17:50:04 +00:00
if err != nil {
2021-02-07 05:14:01 +00:00
log.Print("create error", err)
2020-03-20 17:50:04 +00:00
return err
}
log.Print(resp)
2021-02-07 05:14:01 +00:00
} else {
log.Print(reflect.TypeOf(data).Kind())
2020-03-20 17:50:04 +00:00
return errors.New(INPUT_TYPE_ERROR)
}
2021-02-07 05:14:01 +00:00
} else {
2020-03-20 17:50:04 +00:00
return errors.New(ERROR_PTR)
2020-03-13 11:44:32 +00:00
}
2020-03-13 09:40:00 +00:00
return nil
}
2021-02-07 05:14:01 +00:00
func (p *ElkEngine) Delete(query elastic.Query, index string) error {
if nil != p {
_, err := p.cli.DeleteByQuery().Index(index).Query(query).
Do(context.Background())
2020-03-20 17:50:04 +00:00
if err != nil {
log.Print(err)
2021-02-07 05:14:01 +00:00
return err
2020-03-21 03:17:24 +00:00
}
2021-02-07 05:14:01 +00:00
} else {
2020-03-20 17:50:04 +00:00
return errors.New(ERROR_PTR)
}
2020-03-24 16:30:55 +00:00
return nil
2020-03-20 17:50:04 +00:00
}
2020-03-21 03:17:24 +00:00
2023-09-26 16:32:29 +00:00
func (p *ElkEngine) QueryHighlight(index string, query elastic.Query, v interface{}, hightlight *elastic.Highlight,
2021-08-01 11:50:48 +00:00
limit int, offset int) ([]string, error) {
if reflect.ValueOf(v).Kind() != reflect.Ptr {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Ptr")
}
if reflect.ValueOf(v).Elem().Kind() != reflect.Slice {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice")
}
if reflect.ValueOf(v).Elem().Type().Elem().Kind() != reflect.Struct {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct")
}
eletype := reflect.ValueOf(v).Elem().Type().Elem()
obj := reflect.ValueOf(v).Elem()
objAdd := make([]reflect.Value, 0)
if nil != p {
if limit == 0 {
2021-08-01 15:33:55 +00:00
res, err := p.cli.Search().Index(index).Query(query).Highlight(hightlight).Do(context.TODO())
2021-08-01 11:50:48 +00:00
if err != nil {
print(err)
return nil, err
}
id := []string{}
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
data, e := vs.Source.MarshalJSON()
if nil != e {
log.Print(e.Error())
}
mapobj := map[string]interface{}{}
2021-08-01 15:33:55 +00:00
mapobj["highlight"] = vs.Highlight["content"]
2021-08-01 11:50:48 +00:00
e = json.Unmarshal(data, &mapobj)
if nil != e {
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj)
if nil != e {
log.Print(e.Error())
}
objAdd = append(objAdd, reflect.ValueOf(obj))
}
return id, nil
} else {
2021-08-01 15:33:55 +00:00
res, err := p.cli.Search(index).Query(query).Highlight(hightlight).Size(limit).From(limit * offset).Do(context.TODO())
2021-08-01 11:50:48 +00:00
if err != nil {
print(err)
return nil, err
}
id := []string{}
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
data, e := vs.Source.MarshalJSON()
2021-08-01 15:33:55 +00:00
// log.Print(string(data))
2021-08-01 11:50:48 +00:00
if nil != e {
log.Print(e.Error())
}
mapobj := map[string]interface{}{}
e = json.Unmarshal(data, &mapobj)
if nil != e {
log.Print(e.Error())
}
2021-08-01 15:33:55 +00:00
mapobj["content"] = ""
mapobj["highlight"] = vs.Highlight["content"]
2021-08-01 11:50:48 +00:00
obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj)
if nil != e {
log.Print(e.Error())
}
objAdd = append(objAdd, reflect.ValueOf(obj))
}
addOp := reflect.Append(obj, objAdd...)
obj.Set(addOp)
return id, nil
}
} else {
return nil, errors.New(ERROR_PTR)
}
}
2021-02-07 09:21:19 +00:00
func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
2021-02-08 04:05:38 +00:00
limit int, offset int) ([]string, error) {
2021-02-07 09:21:19 +00:00
if reflect.ValueOf(v).Kind() != reflect.Ptr {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Ptr")
}
if reflect.ValueOf(v).Elem().Kind() != reflect.Slice {
2021-02-08 04:05:38 +00:00
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice")
2021-02-07 09:21:19 +00:00
}
if reflect.ValueOf(v).Elem().Type().Elem().Kind() != reflect.Struct {
2021-02-08 04:05:38 +00:00
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct")
2021-02-07 09:21:19 +00:00
}
eletype := reflect.ValueOf(v).Elem().Type().Elem()
obj := reflect.ValueOf(v).Elem()
objAdd := make([]reflect.Value, 0)
if nil != p {
if limit == 0 {
res, err := p.cli.Search(index).Query(query).Do(context.Background())
if err != nil {
print(err)
2021-02-08 04:05:38 +00:00
return nil, err
2021-02-07 09:21:19 +00:00
}
id := []string{}
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
2021-02-08 02:16:24 +00:00
data, e := vs.Source.MarshalJSON()
if nil != e {
log.Print(e.Error())
}
mapobj := map[string]interface{}{}
e = json.Unmarshal(data, &mapobj)
if nil != e {
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj)
if nil != e {
log.Print(e.Error())
}
objAdd = append(objAdd, reflect.ValueOf(obj))
2021-02-07 09:21:19 +00:00
}
2021-02-08 04:05:38 +00:00
return id, nil
2021-02-07 09:21:19 +00:00
} else {
res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background())
if err != nil {
2023-09-26 16:32:29 +00:00
log.Print(err.Error())
2021-02-08 04:05:38 +00:00
return nil, err
2021-02-07 09:21:19 +00:00
}
id := []string{}
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
data, e := vs.Source.MarshalJSON()
2021-05-22 17:27:55 +00:00
log.Print(string(data))
2021-02-07 09:21:19 +00:00
if nil != e {
log.Print(e.Error())
}
mapobj := map[string]interface{}{}
e = json.Unmarshal(data, &mapobj)
if nil != e {
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj)
if nil != e {
log.Print(e.Error())
}
objAdd = append(objAdd, reflect.ValueOf(obj))
}
addOp := reflect.Append(obj, objAdd...)
obj.Set(addOp)
2021-02-08 04:05:38 +00:00
return id, nil
2021-02-07 09:21:19 +00:00
}
} else {
2021-02-08 04:05:38 +00:00
return nil, errors.New(ERROR_PTR)
2021-02-07 09:21:19 +00:00
}
}
func (p *ElkEngine) QueryGen(index string, query elastic.Query, typ reflect.Type,
2021-02-07 05:14:01 +00:00
limit int, offset int) ([]interface{}, []string, error) {
2021-02-05 17:46:59 +00:00
rets := []interface{}{}
2021-02-07 05:14:01 +00:00
if nil != p {
if limit == 0 {
2020-03-31 17:07:32 +00:00
res, err := p.cli.Search(index).Query(query).Do(context.Background())
if err != nil {
print(err)
2021-02-07 05:14:01 +00:00
return nil, nil, err
}
2021-02-07 05:14:01 +00:00
2020-03-31 17:07:32 +00:00
id := []string{}
2021-02-07 05:14:01 +00:00
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
2020-03-31 17:07:32 +00:00
}
2021-02-07 05:14:01 +00:00
return rets, id, nil
} else {
res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background())
if err != nil {
print(err)
2021-02-07 05:14:01 +00:00
return nil, nil, err
}
2020-03-31 17:07:32 +00:00
id := []string{}
2021-02-07 05:14:01 +00:00
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
data, e := vs.Source.MarshalJSON()
if nil != e {
2021-02-05 17:46:59 +00:00
log.Print(e.Error())
}
mapobj := map[string]interface{}{}
2021-02-07 05:14:01 +00:00
e = json.Unmarshal(data, &mapobj)
if nil != e {
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(typ, mapobj)
if nil != e {
2021-02-05 17:46:59 +00:00
log.Print(e.Error())
}
2021-02-07 05:14:01 +00:00
rets = append(rets, obj)
2020-03-31 17:07:32 +00:00
}
2021-02-07 05:14:01 +00:00
return rets, id, nil
2020-03-21 03:17:24 +00:00
}
2021-02-07 05:14:01 +00:00
} else {
return nil, nil, errors.New(ERROR_PTR)
2020-03-21 03:17:24 +00:00
}
}
2021-02-07 05:14:01 +00:00
func (p *ElkEngine) Update(index string, types string, id string, data map[string]interface{}) error {
2020-03-21 03:17:24 +00:00
if nil != p {
_, err := p.cli.Update().
Index(index).
Id(id).
Doc(data).
Do(context.Background())
2020-03-21 03:17:24 +00:00
if err != nil {
println(err.Error())
return err
}
}
return errors.New(ERROR_PTR)
}
2021-02-07 05:14:01 +00:00
func (p *ElkEngine) CreateIndex(index string, typemaping string) error {
if nil != p {
exists, err := p.cli.IndexExists(index).Do(context.Background())
if err != nil {
// Handle error
log.Print(err)
return err
}
2021-02-07 05:14:01 +00:00
if exists {
return errors.New(INDEX_EXISTED)
}
2020-04-30 17:54:19 +00:00
createIndex, err := p.cli.CreateIndex(index).Body(typemaping).Do(context.Background())
2020-03-25 04:49:26 +00:00
if err != nil {
log.Print(err)
2020-03-25 04:49:26 +00:00
return err
}
if !createIndex.Acknowledged {
return errors.New("create index error")
2020-03-25 04:49:26 +00:00
// Not acknowledged
}
2021-02-07 05:14:01 +00:00
return nil
2020-03-25 04:49:26 +00:00
}
return errors.New(ERROR_PTR)
}
2021-02-07 05:14:01 +00:00
func (p *ElkEngine) IndexExisted(index string) (bool, error) {
if nil != p {
exists, err := p.cli.IndexExists(index).Do(context.Background())
2021-02-07 05:14:01 +00:00
if exists {
return true, nil
}
if err != nil {
// Handle error
log.Print(err)
2021-02-07 05:14:01 +00:00
return false, err
}
2021-02-07 05:14:01 +00:00
return false, nil
}
2021-02-07 05:14:01 +00:00
return false, nil
}