zcy
/
blog_backend_api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

no message

master
zcy 2021-02-07 17:21:19 +08:00
parent 92c76111b8
commit 17897cc87c
4 changed files with 174 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
db/elasticEngine.go
View File

@ -13,7 +13,7 @@ import (
const ( const (
ERROR_PTR = "null pointer error" ERROR_PTR = "null pointer error"
INPUT_TYPE_ERROR = "wrong input parameter" INPUT_TYPE_ERROR = "wrong input parameter: "
CREATED_ERROR = "create error" CREATED_ERROR = "create error"
DELETE_ERROR = "delete error" DELETE_ERROR = "delete error"
INDEX_EXISTED = "index existed" INDEX_EXISTED = "index existed"
@ -60,7 +60,71 @@ func (p *ElkEngine) Delete(query elastic.Query, index string) error {
return nil return nil
} }
func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type, func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
limit int, offset int) ([]string, error) {
if reflect.ValueOf(v).Kind() != reflect.Ptr {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Ptr")
}
if reflect.ValueOf(v).Elem().Kind() != reflect.Slice {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice")
}
if reflect.ValueOf(v).Elem().Type().Elem().Kind() != reflect.Struct {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct")
}
eletype := reflect.ValueOf(v).Elem().Type().Elem()
obj := reflect.ValueOf(v).Elem()
objAdd := make([]reflect.Value, 0)
if nil != p {
if limit == 0 {
res, err := p.cli.Search(index).Query(query).Do(context.Background())
if err != nil {
print(err)
return nil, err
}
id := []string{}
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
}
return id, nil
} else {
res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background())
if err != nil {
print(err)
return nil, err
}
id := []string{}
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
data, e := vs.Source.MarshalJSON()
if nil != e {
log.Print(e.Error())
}
mapobj := map[string]interface{}{}
e = json.Unmarshal(data, &mapobj)
if nil != e {
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj)
log.Print(obj)
if nil != e {
log.Print(e.Error())
}
objAdd = append(objAdd, reflect.ValueOf(obj))
//v = append(v, obj)
}
addOp := reflect.Append(obj, objAdd...)
obj.Set(addOp)
return id, nil
}
} else {
return nil, errors.New(ERROR_PTR)
}
}
func (p *ElkEngine) QueryGen(index string, query elastic.Query, typ reflect.Type,
limit int, offset int) ([]interface{}, []string, error) { limit int, offset int) ([]interface{}, []string, error) {
rets := []interface{}{} rets := []interface{}{}
if nil != p { if nil != p {
@ -98,9 +162,6 @@ func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type,
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
// for k,_ := range mapobj{
// value := reflect.ValueOf(obj)
// }
rets = append(rets, obj) rets = append(rets, obj)
} }
return rets, id, nil return rets, id, nil

73
model/hardware.go
View File

@ -14,7 +14,7 @@ const (
ERR_COLUMN_EXISTED = "column_existed" ERR_COLUMN_EXISTED = "column_existed"
) )
func HardwareTypeMapping() (string){ func HardwareTypeMapping() string {
return `"mappings":{ return `"mappings":{
"hardware":{ "hardware":{
"properties":{ "properties":{
@ -30,74 +30,71 @@ func HardwareTypeMapping() (string){
// this api is based on elasticsearch // this api is based on elasticsearch
type Hardware struct { type Hardware struct {
ID string `json:"_id,omitempty"` ID string `json:"_id,omitempty"`
BuyDate string `json:"buy_date,omitempty"` //购入时间 BuyDate string `json:"buy_date,omitempty"` //购入时间
Name string `json:"name,omitempty"` // 名字 Name string `json:"name,omitempty"` // 名字
Desc string `json:"desc,omitempty"` // 描述 Desc string `json:"desc,omitempty"` // 描述
Pic string `json:"pic,omitempty"` // 图片 Pic string `json:"pic,omitempty"` // 图片
Doc string `json:"doc,omitempty"` //文档资料 Doc string `json:"doc,omitempty"` //文档资料
} }
func (this *Hardware )CreateHardware( ) error{ func (this *Hardware) CreateHardware() error {
if nil == this{ if nil == this {
return errors.New(utils.ERRNULLPOINTER) return errors.New(utils.ERRNULLPOINTER)
} }
log.Print(this.Name) log.Print(this.Name)
matchPhraseQuery := elastic.NewMatchQuery("name", this.Name) matchPhraseQuery := elastic.NewMatchQuery("name", this.Name)
existedHardware,e := QueryHardwares(matchPhraseQuery,10,0) existedHardware, e := QueryHardwares(matchPhraseQuery, 10, 0)
log.Print(e,existedHardware) log.Print(e, existedHardware)
for _,v := range existedHardware{ for _, v := range existedHardware {
if v.Name == this.Name{ if v.Name == this.Name {
log.Print(v.ID) log.Print(v.ID)
return errors.New(ERR_COLUMN_EXISTED) return errors.New(ERR_COLUMN_EXISTED)
} }
} }
e = db.GetElastic().Create("hardware_data","0","",*this) e = db.GetElastic().Create("hardware_data", "0", "", *this)
if nil != e{ if nil != e {
log.Print("shit1") log.Print("shit1")
log.Print(e.Error()) log.Print(e.Error())
return e return e
} }
log.Print("shit2") log.Print("shit2")
return nil; return nil
} }
func GetHardwares(limit int,size int) ([]Hardware,error){ func GetHardwares(limit int, size int) ([]Hardware, error) {
var ret []Hardware var ret []Hardware
data,ids,e := db.GetElastic().Query("hardware_data",nil,reflect.TypeOf(Hardware{}),limit,size) ids, e := db.GetElastic().Query("hardware_data", nil, reflect.TypeOf(Hardware{}), limit, size)
if nil != e{ if nil != e {
return nil,e return nil, e
} }
i := 0 i := 0
for _,v := range data{ for _, v := range ids {
ret = append(ret,v.(Hardware)) ret[i].ID = v
ret[i].ID = ids[i]
i++ i++
} }
return ret,nil return ret, nil
} }
func QueryHardwares(query elastic.Query,limit int,offset int) ([]Hardware,error){ func QueryHardwares(query elastic.Query, limit int, offset int) ([]Hardware, error) {
var ret []Hardware ret := []Hardware{}
data,ids,e := db.GetElastic().Query("hardware_data",query,reflect.TypeOf(Hardware{}),limit,offset) ids, e := db.GetElastic().Query("hardware_data", query, ret, limit, offset)
log.Print(data) if nil != e {
if nil != e{ return nil, e
return nil,e
} }
i := 0 i := 0
for _,v := range data{ for _, v := range ids {
ret = append(ret,v.(Hardware)) ret[i].ID = v
ret[i].ID = ids[i]
i++ i++
} }
return ret,nil return ret, nil
} }
func DeleteHardware(name string) error{ func DeleteHardware(name string) error {
query := elastic.NewTermQuery("name",name) query := elastic.NewTermQuery("name", name)
err := db.GetElastic().Delete(query,"hardware_data") err := db.GetElastic().Delete(query, "hardware_data")
if err != nil{ if err != nil {
return err return err
} }
return nil return nil

101
model/port.go
View File

@ -2,7 +2,6 @@ package model
import ( import (
"background/db" "background/db"
"reflect"
"strings" "strings"
"ubntgo/logger" "ubntgo/logger"
@ -13,30 +12,30 @@ import (
) )
type Field struct { type Field struct {
Field string `sql:"Field"` Field string `sql:"Field"`
Type string `sql:"Type"` Type string `sql:"Type"`
Key string `sql:"Key"` Key string `sql:"Key"`
} }
func MysqlToElasticSearchMapping(types string, Key string) string {
func MysqlToElasticSearchMapping(types string,Key string) string{ if Key == "PRI" {
if Key == "PRI"{
return "keyword" return "keyword"
} }
if(strings.Contains(types,"int(")){ if strings.Contains(types, "int(") {
return "integer" return "integer"
} }
if(strings.Contains(types,"longblob")){ if strings.Contains(types, "longblob") {
return "text" return "text"
} }
if(strings.Contains(types,"varchar")){ if strings.Contains(types, "varchar") {
return "text" return "text"
} }
if(strings.Contains(types,"datetime")){ if strings.Contains(types, "datetime") {
return "date" return "date"
} }
return "" return ""
} }
/* /*
"settings":{ "settings":{
"number_of_shards":1, "number_of_shards":1,
@ -65,99 +64,97 @@ func MysqlToElasticSearchMapping(types string,Key string) string{
} }
*/ */
// 不同类型db之间进行缓存 // 不同类型db之间进行缓存
func QueryDocument(query elastic.Query,limit int,offset int) ([]Hardware,error){ func QueryDocument(query elastic.Query, limit int, offset int) ([]Hardware, error) {
var ret []Hardware var ret []Hardware
data,ids,e := db.GetElastic().Query("doc",query,reflect.TypeOf(Hardware{}),limit,offset) ids, e := db.GetElastic().Query("doc", query, ret, limit, offset)
log.Print(data) if nil != e {
if nil != e{ return nil, e
return nil,e
} }
i := 0 i := 0
for _,v := range data{ for _, v := range ids {
ret = append(ret,v.(Hardware)) ret[i].ID = v
ret[i].ID = ids[i]
i++ i++
} }
return ret,nil return ret, nil
} }
func InsertDocToElaticSearch(doc Doc) error{ func InsertDocToElaticSearch(doc Doc) error {
matchPhraseQuery := elastic.NewMatchQuery("title", doc.Title) matchPhraseQuery := elastic.NewMatchQuery("title", doc.Title)
existedHardware,e := QueryHardwares(matchPhraseQuery,10,0) existedHardware, e := QueryHardwares(matchPhraseQuery, 10, 0)
log.Print(e,existedHardware) log.Print(e, existedHardware)
for _,v := range existedHardware{ for _, v := range existedHardware {
if v.Name == doc.Title{ if v.Name == doc.Title {
log.Print(v.ID) log.Print(v.ID)
return errors.New(200,"existed title") return errors.New(200, "existed title")
} }
} }
e = db.GetElastic().Create("doc","0","",doc) e = db.GetElastic().Create("doc", "0", "", doc)
if nil != e{ if nil != e {
log.Print(e.Error()) log.Print(e.Error())
return e return e
} }
return nil return nil
} }
func PortDocumentToElasticsearch(tblname string ) error{ func PortDocumentToElasticsearch(tblname string) error {
ret,e := GetAllDocs() ret, e := GetAllDocs()
if nil != e{ if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
insert,err := json.Marshal(ret) insert, err := json.Marshal(ret)
if nil != err{ if nil != err {
log.Print(err) log.Print(err)
} }
log.Print(len(ret)) log.Print(len(ret))
log.Print(string(insert)) log.Print(string(insert))
columns := []Field{} columns := []Field{}
e = db.GetMysqlClient().Query2("describe " + tblname,&columns) e = db.GetMysqlClient().Query2("describe "+tblname, &columns)
if nil != e{ if nil != e {
logger.Debug(e.Error()) logger.Debug(e.Error())
return e return e
} }
if existed,_ := db.GetElastic().IndexExisted(tblname);existed{ if existed, _ := db.GetElastic().IndexExisted(tblname); existed {
for _,v := range ret{ for _, v := range ret {
e := InsertDocToElaticSearch(v) e := InsertDocToElaticSearch(v)
if nil != e{ if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
log.Print("index not existed , create " + tblname) log.Print("index not existed , create " + tblname)
} }
}else{ } else {
props := map[string]interface{}{} props := map[string]interface{}{}
mapping := map[string]interface{}{ mapping := map[string]interface{}{
"settings":map[string]interface{}{ "settings": map[string]interface{}{
"analysis":map[string]interface{}{ "analysis": map[string]interface{}{
"analyzer":map[string]interface{}{ "analyzer": map[string]interface{}{
"default":map[string]interface{}{ "default": map[string]interface{}{
"type": "smartcn", "type": "smartcn",
}, },
}, },
}, },
}, },
"mappings": map[string]interface{}{ "mappings": map[string]interface{}{
"properties":props, "properties": props,
}, },
} }
for _,v := range columns{ for _, v := range columns {
props[v.Field] = map[string]string{"type":MysqlToElasticSearchMapping(v.Type,v.Key)}; props[v.Field] = map[string]string{"type": MysqlToElasticSearchMapping(v.Type, v.Key)}
} }
dat,e := json.Marshal(mapping) dat, e := json.Marshal(mapping)
if nil != e{ if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
e = db.GetElastic().CreateIndex(tblname,string(dat)) e = db.GetElastic().CreateIndex(tblname, string(dat))
if nil != e{ if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
log.Print(string(dat)) log.Print(string(dat))
for _,v := range ret{ for _, v := range ret {
e := InsertDocToElaticSearch(v) e := InsertDocToElaticSearch(v)
if nil != e{ if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
} }

28
test/portData_test.go
View File

@ -6,6 +6,7 @@ import (
"background/logs" "background/logs"
"background/model" "background/model"
"background/utils" "background/utils"
"fmt"
"log" "log"
"reflect" "reflect"
"testing" "testing"
@ -19,6 +20,7 @@ func InitConfig() {
log.Println(e.Error()) log.Println(e.Error())
} }
} }
func InitMysql() { func InitMysql() {
c := config.GetMysqlConfig() c := config.GetMysqlConfig()
if c == nil { if c == nil {
@ -27,6 +29,7 @@ func InitMysql() {
db.Init() db.Init()
} }
} }
func InitRedisConfig() { func InitRedisConfig() {
e := config.InitRedis() e := config.InitRedis()
if nil != e { if nil != e {
@ -34,6 +37,7 @@ func InitRedisConfig() {
return return
} }
} }
func InitElasticSearch() { func InitElasticSearch() {
e := db.GetElastic().CreateIndex("hardware", model.HardwareTypeMapping()) e := db.GetElastic().CreateIndex("hardware", model.HardwareTypeMapping())
if nil != e { if nil != e {
@ -77,18 +81,30 @@ func TestQueryDoc(t *testing.T) {
db.InitELK() db.InitELK()
query := elastic.NewTermQuery("content", "title") query := elastic.NewTermQuery("content", "title")
searchResult, titles, e := db.GetElastic().Query("doc", query, reflect.TypeOf(model.Doc{}), 10, 0) x := []model.Doc{}
titles, e := db.GetElastic().Query("doc", query, &x, 10, 0)
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
log.Print(searchResult) log.Print(x)
log.Print(titles) log.Print(titles)
} }
func TestChangeStructFieldThroughStruct(t *testing.T) { func TestChangeStructFieldThroughStruct(t *testing.T) {
type XX struct { var arr1 = []int{1, 2, 3}
A int16 `json:"bb"` log.Print(reflect.ValueOf(&arr1).Kind())
B string `json: "cc" xml:"test"` a0 := reflect.ValueOf(&arr1).Elem()
} log.Print(a0.Kind())
e0 := make([]reflect.Value, 0)
e0 = append(e0, reflect.ValueOf(100))
e0 = append(e0, reflect.ValueOf(200))
e0 = append(e0, reflect.ValueOf(300))
e0 = append(e0, reflect.ValueOf(400))
val_arr1 := reflect.Append(a0, e0...)
a0.Set(val_arr1)
fmt.Println("a0 is ", a0)
fmt.Println("arr1 is ", arr1)
fmt.Println(cap(e0))
fmt.Println(cap(arr1))
} }