zcy
/
blog_backend_api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

no message

master
zcy 2021-02-08 12:05:38 +08:00
parent ce21b97dba
commit c594771faf
6 changed files with 105 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
db/elasticEngine.go
View File

@ -61,16 +61,16 @@ func (p *ElkEngine) Delete(query elastic.Query, index string) error {
}
func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
limit int, offset int) ( []string, error) {
limit int, offset int) ([]string, error) {
if reflect.ValueOf(v).Kind() != reflect.Ptr {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Ptr")
}
if reflect.ValueOf(v).Elem().Kind() != reflect.Slice {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice")
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice")
}
if reflect.ValueOf(v).Elem().Type().Elem().Kind() != reflect.Struct {
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct")
return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct")
}
eletype := reflect.ValueOf(v).Elem().Type().Elem()
obj := reflect.ValueOf(v).Elem()
@ -81,7 +81,7 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
res, err := p.cli.Search(index).Query(query).Do(context.Background())
if err != nil {
print(err)
return nil, err
return nil, err
}
id := []string{}
@ -103,12 +103,12 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
}
objAdd = append(objAdd, reflect.ValueOf(obj))
}
return id, nil
return id, nil
} else {
res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background())
if err != nil {
print(err)
return nil, err
return nil, err
}
id := []string{}
for _, vs := range res.Hits.Hits {
@ -123,7 +123,6 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj)
log.Print(obj)
if nil != e {
log.Print(e.Error())
}
@ -131,10 +130,10 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
}
addOp := reflect.Append(obj, objAdd...)
obj.Set(addOp)
return id, nil
return id, nil
}
} else {
return nil, errors.New(ERROR_PTR)
return nil, errors.New(ERROR_PTR)
}
}

10
model/blog.go
View File

@ -52,6 +52,7 @@ func GetArticlesType() []ArticleType {
}
return ret
}
/*
CreateDoc 新建文档
*/
@ -67,7 +68,7 @@ func CreateDoc(doc Doc) error {
DUAL
WHERE
NOT EXISTS ( SELECT * FROM doc WHERE doc.title = '%s' );`, doc.Title, strings.Replace(doc.Content, "'", "\\'", -1),
doc.Author, doc.Type, doc.IsPublic,time.Now().Format("2006-01-02 15:04:05"), doc.Title)
doc.Author, doc.Type, doc.IsPublic, time.Now().Format("2006-01-02 15:04:05"), doc.Title)
_, e := db.GetMysqlClient().Query(sql)
if nil != e {
log.Print(sql)
@ -76,13 +77,14 @@ func CreateDoc(doc Doc) error {
}
return nil
}
/*
UpdateDoc 更新文档
*/
func UpdateDoc(doc Doc) error {
sql := fmt.Sprintf(`update doc set doc.author = '%s' ,doc.title = '%s',doc.type = '%d',doc.content = '%s' ,doc.update_time = '%s' where doc.id = '%d'; `,
doc.Author, doc.Title, doc.Type,
strings.Replace(doc.Content, "'", "\\'", -1), time.Now().Format("2006-01-02 15:04:05"),doc.ID)
strings.Replace(doc.Content, "'", "\\'", -1), time.Now().Format("2006-01-02 15:04:05"), doc.ID)
_, e := db.GetMysqlClient().Query(sql)
if nil != e {
logs.Error(e.Error())
@ -90,6 +92,7 @@ func UpdateDoc(doc Doc) error {
}
return nil
}
/*
DeleteDoc 删除文档
*/
@ -102,6 +105,7 @@ func DeleteDoc(id int64) error {
}
return nil
}
/*
AddArticleType 添加文档类型
*/
@ -115,6 +119,7 @@ func AddArticleType(t ArticleType) error {
}
return nil
}
/*
UpdateArticleType 更新文档类型
*/
@ -128,6 +133,7 @@ func UpdateArticleType(t ArticleType) error {
return nil
}
/*
DeleteArticleType 删除文档类型
*/

43
model/port.go
View File

@ -97,6 +97,49 @@ func InsertDocToElaticSearch(doc Doc) error {
return nil
}
func CreateIndexFromMysqlTable(tblname string) error {
columns := []Field{}
e := db.GetMysqlClient().Query2("describe "+tblname, &columns)
if nil != e {
logger.Debug(e.Error())
return e
}
if existed, _ := db.GetElastic().IndexExisted(tblname); existed {
log.Print("index not existed , create " + tblname)
} else {
props := map[string]interface{}{}
mapping := map[string]interface{}{
"settings": map[string]interface{}{
"analysis": map[string]interface{}{
"analyzer": map[string]interface{}{
"default": map[string]interface{}{
"type": "smartcn",
},
},
},
},
"mappings": map[string]interface{}{
"properties": props,
},
}
for _, v := range columns {
props[v.Field] = map[string]string{"type": MysqlToElasticSearchMapping(v.Type, v.Key)}
}
dat, e := json.Marshal(mapping)
if nil != e {
log.Print(e.Error())
}
e = db.GetElastic().CreateIndex(tblname, string(dat))
if nil != e {
log.Print(e.Error())
}
log.Print(string(dat))
}
return nil
}
func PortDocumentToElasticsearch(tblname string) error {
ret, e := GetAllDocs()
if nil != e {

1
test/git_test.go
View File

@ -3,6 +3,7 @@ package test
import (
"log"
"testing"
"github.com/sergi/go-diff/diffmatchpatch"
)

25
test/portData_test.go
View File

@ -47,6 +47,9 @@ func InitElasticSearch() {
func InitLogs() {
logs.Init(config.GetLogConfig().Dir, config.GetLogConfig().File, config.GetLogConfig().Level, config.GetLogConfig().SaveFile)
}
// 从mysql数据自动导出数据到es，从describe 表新建索引和mapping，然后全表导出到es
// port data from mysql to es, it can describe table and create index with mapping ,then port all data to es.
func TestPortDocToElastic(t *testing.T) {
InitConfig()
InitLogs()
@ -73,6 +76,19 @@ func TestReflect(t *testing.T) {
log.Print(xx)
}
// 测试新建索引,查看mapping是否生效
// test create new index,and check the mapping works
func TestCreateIndex(t *testing.T) {
InitConfig()
InitLogs()
InitRedisConfig()
InitMysql()
db.InitELK()
InitElasticSearch()
}
// 测试query doc
// test doc query works
func TestQueryDoc(t *testing.T) {
InitConfig()
InitLogs()
@ -80,14 +96,15 @@ func TestQueryDoc(t *testing.T) {
InitMysql()
db.InitELK()
query := elastic.NewTermQuery("content", "title")
query := elastic.NewTermQuery("title", "c")
x := []model.Doc{}
titles, e := db.GetElastic().Query("doc", query, &x, 10, 0)
_, e := db.GetElastic().Query("doc", query, &x, 10, 0)
if nil != e {
log.Print(e.Error())
}
log.Print(x)
log.Print(titles)
for _, v := range x {
log.Print(v.Title, " ", v.ID, " ", v.Type)
}
}
func TestChangeStructFieldThroughStruct(t *testing.T) {

26
utils/reflect.go
View File

@ -70,6 +70,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Int8:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
@ -77,6 +80,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Int16:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
@ -84,6 +90,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Int32:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
@ -91,6 +100,19 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Int64:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Uint:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
@ -234,9 +256,9 @@ func UnmarshalJson2StructGen(t reflect.Type, maps map[string]interface{}) (inter
for k, v := range maps {
log.Print(k)
if filedName, ok := remap["json"][k]; ok {
// log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
// log.Print(filedName, " ", opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) {
// log.Print(opStruct.FieldByName(filedName).CanAddr(), opStruct.FieldByName(filedName).CanSet())
// log.Print(filedName, " can set ", opStruct.FieldByName(filedName).CanSet(), "can addr ", opStruct.FieldByName(filedName).CanAddr())
if opStruct.FieldByName(filedName).CanSet() {
opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type()))
}