zcy
/
blog_backend_api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

no message

master
zcy 2021-02-07 17:21:19 +08:00
parent 92c76111b8
commit 17897cc87c
4 changed files with 174 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
db/elasticEngine.go
View File

@ -13,7 +13,7 @@ import (
const (
ERROR_PTR = "null pointer error"
INPUT_TYPE_ERROR = "wrong input parameter"
INPUT_TYPE_ERROR = "wrong input parameter: "
CREATED_ERROR = "create error"
DELETE_ERROR = "delete error"
INDEX_EXISTED = "index existed"
@ -60,7 +60,71 @@ func (p *ElkEngine) Delete(query elastic.Query, index string) error {
return nil
}
func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type,
func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
limit int, offset int) ([]string, error) {
if reflect.ValueOf(v).Kind() != reflect.Ptr {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Ptr")
}
if reflect.ValueOf(v).Elem().Kind() != reflect.Slice {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice")
}
if reflect.ValueOf(v).Elem().Type().Elem().Kind() != reflect.Struct {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct")
}
eletype := reflect.ValueOf(v).Elem().Type().Elem()
obj := reflect.ValueOf(v).Elem()
objAdd := make([]reflect.Value, 0)
if nil != p {
if limit == 0 {
res, err := p.cli.Search(index).Query(query).Do(context.Background())
if err != nil {
print(err)
return nil, err
}
id := []string{}
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
}
return id, nil
} else {
res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background())
if err != nil {
print(err)
return nil, err
}
id := []string{}
for _, vs := range res.Hits.Hits {
id = append(id, vs.Id)
data, e := vs.Source.MarshalJSON()
if nil != e {
log.Print(e.Error())
}
mapobj := map[string]interface{}{}
e = json.Unmarshal(data, &mapobj)
if nil != e {
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj)
log.Print(obj)
if nil != e {
log.Print(e.Error())
}
objAdd = append(objAdd, reflect.ValueOf(obj))
//v = append(v, obj)
}
addOp := reflect.Append(obj, objAdd...)
obj.Set(addOp)
return id, nil
}
} else {
return nil, errors.New(ERROR_PTR)
}
}
func (p *ElkEngine) QueryGen(index string, query elastic.Query, typ reflect.Type,
limit int, offset int) ([]interface{}, []string, error) {
rets := []interface{}{}
if nil != p {
@ -98,9 +162,6 @@ func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type,
if nil != e {
log.Print(e.Error())
}
// for k,_ := range mapobj{
// value := reflect.ValueOf(obj)
// }
rets = append(rets, obj)
}
return rets, id, nil

75
model/hardware.go
View File

@ -14,7 +14,7 @@ const (
ERR_COLUMN_EXISTED = "column_existed"
)
func HardwareTypeMapping() (string){
func HardwareTypeMapping() string {
return `"mappings":{
"hardware":{
"properties":{
@ -30,75 +30,72 @@ func HardwareTypeMapping() (string){
// this api is based on elasticsearch
type Hardware struct {
ID string `json:"_id,omitempty"`
BuyDate string `json:"buy_date,omitempty"` //购入时间
Name string `json:"name,omitempty"` // 名字
Desc string `json:"desc,omitempty"` // 描述
Pic string `json:"pic,omitempty"` // 图片
Doc string `json:"doc,omitempty"` //文档资料
ID string `json:"_id,omitempty"`
BuyDate string `json:"buy_date,omitempty"` //购入时间
Name string `json:"name,omitempty"` // 名字
Desc string `json:"desc,omitempty"` // 描述
Pic string `json:"pic,omitempty"` // 图片
Doc string `json:"doc,omitempty"` //文档资料
}
func (this *Hardware )CreateHardware( ) error{
if nil == this{
func (this *Hardware) CreateHardware() error {
if nil == this {
return errors.New(utils.ERRNULLPOINTER)
}
log.Print(this.Name)
matchPhraseQuery := elastic.NewMatchQuery("name", this.Name)
existedHardware,e := QueryHardwares(matchPhraseQuery,10,0)
log.Print(e,existedHardware)
existedHardware, e := QueryHardwares(matchPhraseQuery, 10, 0)
log.Print(e, existedHardware)
for _,v := range existedHardware{
if v.Name == this.Name{
for _, v := range existedHardware {
if v.Name == this.Name {
log.Print(v.ID)
return errors.New(ERR_COLUMN_EXISTED)
}
}
e = db.GetElastic().Create("hardware_data","0","",*this)
if nil != e{
e = db.GetElastic().Create("hardware_data", "0", "", *this)
if nil != e {
log.Print("shit1")
log.Print(e.Error())
return e
}
log.Print("shit2")
return nil;
return nil
}
func GetHardwares(limit int,size int) ([]Hardware,error){
func GetHardwares(limit int, size int) ([]Hardware, error) {
var ret []Hardware
data,ids,e := db.GetElastic().Query("hardware_data",nil,reflect.TypeOf(Hardware{}),limit,size)
if nil != e{
return nil,e
ids, e := db.GetElastic().Query("hardware_data", nil, reflect.TypeOf(Hardware{}), limit, size)
if nil != e {
return nil, e
}
i := 0
for _,v := range data{
ret = append(ret,v.(Hardware))
ret[i].ID = ids[i]
for _, v := range ids {
ret[i].ID = v
i++
}
return ret,nil
return ret, nil
}
func QueryHardwares(query elastic.Query,limit int,offset int) ([]Hardware,error){
var ret []Hardware
data,ids,e := db.GetElastic().Query("hardware_data",query,reflect.TypeOf(Hardware{}),limit,offset)
log.Print(data)
if nil != e{
return nil,e
func QueryHardwares(query elastic.Query, limit int, offset int) ([]Hardware, error) {
ret := []Hardware{}
ids, e := db.GetElastic().Query("hardware_data", query, ret, limit, offset)
if nil != e {
return nil, e
}
i := 0
for _,v := range data{
ret = append(ret,v.(Hardware))
ret[i].ID = ids[i]
for _, v := range ids {
ret[i].ID = v
i++
}
return ret,nil
return ret, nil
}
func DeleteHardware(name string) error{
query := elastic.NewTermQuery("name",name)
err := db.GetElastic().Delete(query,"hardware_data")
if err != nil{
func DeleteHardware(name string) error {
query := elastic.NewTermQuery("name", name)
err := db.GetElastic().Delete(query, "hardware_data")
if err != nil {
return err
}
return nil
}
}

103
model/port.go
View File

@ -2,7 +2,6 @@ package model
import (
"background/db"
"reflect"
"strings"
"ubntgo/logger"
@ -13,30 +12,30 @@ import (
)
type Field struct {
Field string `sql:"Field"`
Type string `sql:"Type"`
Key string `sql:"Key"`
Field string `sql:"Field"`
Type string `sql:"Type"`
Key string `sql:"Key"`
}
func MysqlToElasticSearchMapping(types string,Key string) string{
if Key == "PRI"{
func MysqlToElasticSearchMapping(types string, Key string) string {
if Key == "PRI" {
return "keyword"
}
if(strings.Contains(types,"int(")){
if strings.Contains(types, "int(") {
return "integer"
}
if(strings.Contains(types,"longblob")){
if strings.Contains(types, "longblob") {
return "text"
}
if(strings.Contains(types,"varchar")){
if strings.Contains(types, "varchar") {
return "text"
}
if(strings.Contains(types,"datetime")){
if strings.Contains(types, "datetime") {
return "date"
}
return ""
}
/*
"settings":{
"number_of_shards":1,
@ -65,103 +64,101 @@ func MysqlToElasticSearchMapping(types string,Key string) string{
}
*/
// 不同类型db之间进行缓存
func QueryDocument(query elastic.Query,limit int,offset int) ([]Hardware,error){
func QueryDocument(query elastic.Query, limit int, offset int) ([]Hardware, error) {
var ret []Hardware
data,ids,e := db.GetElastic().Query("doc",query,reflect.TypeOf(Hardware{}),limit,offset)
log.Print(data)
if nil != e{
return nil,e
ids, e := db.GetElastic().Query("doc", query, ret, limit, offset)
if nil != e {
return nil, e
}
i := 0
for _,v := range data{
ret = append(ret,v.(Hardware))
ret[i].ID = ids[i]
for _, v := range ids {
ret[i].ID = v
i++
}
return ret,nil
return ret, nil
}
func InsertDocToElaticSearch(doc Doc) error{
func InsertDocToElaticSearch(doc Doc) error {
matchPhraseQuery := elastic.NewMatchQuery("title", doc.Title)
existedHardware,e := QueryHardwares(matchPhraseQuery,10,0)
log.Print(e,existedHardware)
existedHardware, e := QueryHardwares(matchPhraseQuery, 10, 0)
log.Print(e, existedHardware)
for _,v := range existedHardware{
if v.Name == doc.Title{
for _, v := range existedHardware {
if v.Name == doc.Title {
log.Print(v.ID)
return errors.New(200,"existed title")
return errors.New(200, "existed title")
}
}
e = db.GetElastic().Create("doc","0","",doc)
if nil != e{
e = db.GetElastic().Create("doc", "0", "", doc)
if nil != e {
log.Print(e.Error())
return e
}
return nil
}
func PortDocumentToElasticsearch(tblname string ) error{
ret,e := GetAllDocs()
if nil != e{
func PortDocumentToElasticsearch(tblname string) error {
ret, e := GetAllDocs()
if nil != e {
log.Print(e.Error())
}
insert,err := json.Marshal(ret)
if nil != err{
insert, err := json.Marshal(ret)
if nil != err {
log.Print(err)
}
log.Print(len(ret))
log.Print(string(insert))
columns := []Field{}
e = db.GetMysqlClient().Query2("describe " + tblname,&columns)
if nil != e{
e = db.GetMysqlClient().Query2("describe "+tblname, &columns)
if nil != e {
logger.Debug(e.Error())
return e
}
if existed,_ := db.GetElastic().IndexExisted(tblname);existed{
for _,v := range ret{
if existed, _ := db.GetElastic().IndexExisted(tblname); existed {
for _, v := range ret {
e := InsertDocToElaticSearch(v)
if nil != e{
if nil != e {
log.Print(e.Error())
}
log.Print("index not existed , create " + tblname)
}
}else{
} else {
props := map[string]interface{}{}
mapping := map[string]interface{}{
"settings":map[string]interface{}{
"analysis":map[string]interface{}{
"analyzer":map[string]interface{}{
"default":map[string]interface{}{
"settings": map[string]interface{}{
"analysis": map[string]interface{}{
"analyzer": map[string]interface{}{
"default": map[string]interface{}{
"type": "smartcn",
},
},
},
},
"mappings": map[string]interface{}{
"properties":props,
"properties": props,
},
}
for _,v := range columns{
props[v.Field] = map[string]string{"type":MysqlToElasticSearchMapping(v.Type,v.Key)};
for _, v := range columns {
props[v.Field] = map[string]string{"type": MysqlToElasticSearchMapping(v.Type, v.Key)}
}
dat,e := json.Marshal(mapping)
if nil != e{
dat, e := json.Marshal(mapping)
if nil != e {
log.Print(e.Error())
}
e = db.GetElastic().CreateIndex(tblname,string(dat))
if nil != e{
e = db.GetElastic().CreateIndex(tblname, string(dat))
if nil != e {
log.Print(e.Error())
}
log.Print(string(dat))
for _,v := range ret{
for _, v := range ret {
e := InsertDocToElaticSearch(v)
if nil != e{
if nil != e {
log.Print(e.Error())
}
}
}
return nil
}
}

28
test/portData_test.go
View File

@ -6,6 +6,7 @@ import (
"background/logs"
"background/model"
"background/utils"
"fmt"
"log"
"reflect"
"testing"
@ -19,6 +20,7 @@ func InitConfig() {
log.Println(e.Error())
}
}
func InitMysql() {
c := config.GetMysqlConfig()
if c == nil {
@ -27,6 +29,7 @@ func InitMysql() {
db.Init()
}
}
func InitRedisConfig() {
e := config.InitRedis()
if nil != e {
@ -34,6 +37,7 @@ func InitRedisConfig() {
return
}
}
func InitElasticSearch() {
e := db.GetElastic().CreateIndex("hardware", model.HardwareTypeMapping())
if nil != e {
@ -77,18 +81,30 @@ func TestQueryDoc(t *testing.T) {
db.InitELK()
query := elastic.NewTermQuery("content", "title")
searchResult, titles, e := db.GetElastic().Query("doc", query, reflect.TypeOf(model.Doc{}), 10, 0)
x := []model.Doc{}
titles, e := db.GetElastic().Query("doc", query, &x, 10, 0)
if nil != e {
log.Print(e.Error())
}
log.Print(searchResult)
log.Print(x)
log.Print(titles)
}
func TestChangeStructFieldThroughStruct(t *testing.T) {
type XX struct {
A int16 `json:"bb"`
B string `json: "cc" xml:"test"`
}
var arr1 = []int{1, 2, 3}
log.Print(reflect.ValueOf(&arr1).Kind())
a0 := reflect.ValueOf(&arr1).Elem()
log.Print(a0.Kind())
e0 := make([]reflect.Value, 0)
e0 = append(e0, reflect.ValueOf(100))
e0 = append(e0, reflect.ValueOf(200))
e0 = append(e0, reflect.ValueOf(300))
e0 = append(e0, reflect.ValueOf(400))
val_arr1 := reflect.Append(a0, e0...)
a0.Set(val_arr1)
fmt.Println("a0 is ", a0)
fmt.Println("arr1 is ", arr1)
fmt.Println(cap(e0))
fmt.Println(cap(arr1))
}