caiyu/background
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

no message

Browse Source
This commit is contained in:
zcy 2021-02-08 12:05:38 +08:00
parent ce21b97dba
commit c594771faf
6 changed files with 105 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
db/elasticEngine.go
View File

@ -123,7 +123,6 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
log.Print(e.Error())
}
obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj)
log.Print(obj)
if nil != e {
log.Print(e.Error())
}

6
model/blog.go
View File

@ -52,6 +52,7 @@ func GetArticlesType() []ArticleType {
}
return ret
}
/*
CreateDoc 新建文档
*/
@ -76,6 +77,7 @@ func CreateDoc(doc Doc) error {
}
return nil
}
/*
UpdateDoc 更新文档
*/
@ -90,6 +92,7 @@ func UpdateDoc(doc Doc) error {
}
return nil
}
/*
DeleteDoc 删除文档
*/
@ -102,6 +105,7 @@ func DeleteDoc(id int64) error {
}
return nil
}
/*
AddArticleType 添加文档类型
*/
@ -115,6 +119,7 @@ func AddArticleType(t ArticleType) error {
}
return nil
}
/*
UpdateArticleType 更新文档类型
*/
@ -128,6 +133,7 @@ func UpdateArticleType(t ArticleType) error {
return nil
}
/*
DeleteArticleType 删除文档类型
*/

43
model/port.go
View File

@ -97,6 +97,49 @@ func InsertDocToElaticSearch(doc Doc) error {
return nil
}
func CreateIndexFromMysqlTable(tblname string) error {
columns := []Field{}
e := db.GetMysqlClient().Query2("describe "+tblname, &columns)
if nil != e {
logger.Debug(e.Error())
return e
}
if existed, _ := db.GetElastic().IndexExisted(tblname); existed {
log.Print("index not existed , create " + tblname)
} else {
props := map[string]interface{}{}
mapping := map[string]interface{}{
"settings": map[string]interface{}{
"analysis": map[string]interface{}{
"analyzer": map[string]interface{}{
"default": map[string]interface{}{
"type": "smartcn",
},
},
},
},
"mappings": map[string]interface{}{
"properties": props,
},
}
for _, v := range columns {
props[v.Field] = map[string]string{"type": MysqlToElasticSearchMapping(v.Type, v.Key)}
}
dat, e := json.Marshal(mapping)
if nil != e {
log.Print(e.Error())
}
e = db.GetElastic().CreateIndex(tblname, string(dat))
if nil != e {
log.Print(e.Error())
}
log.Print(string(dat))
}
return nil
}
func PortDocumentToElasticsearch(tblname string) error {
ret, e := GetAllDocs()
if nil != e {

1
test/git_test.go
View File

@ -3,6 +3,7 @@ package test
import (
"log"
"testing"
"github.com/sergi/go-diff/diffmatchpatch"
)

25
test/portData_test.go
View File

@ -47,6 +47,9 @@ func InitElasticSearch() {
func InitLogs() {
logs.Init(config.GetLogConfig().Dir, config.GetLogConfig().File, config.GetLogConfig().Level, config.GetLogConfig().SaveFile)
}
// 从mysql数据自动导出数据到es，从describe 表新建索引和mapping，然后全表导出到es
// port data from mysql to es, it can describe table and create index with mapping ,then port all data to es.
func TestPortDocToElastic(t *testing.T) {
InitConfig()
InitLogs()
@ -73,6 +76,19 @@ func TestReflect(t *testing.T) {
log.Print(xx)
}
// 测试新建索引,查看mapping是否生效
// test create new index,and check the mapping works
func TestCreateIndex(t *testing.T) {
InitConfig()
InitLogs()
InitRedisConfig()
InitMysql()
db.InitELK()
InitElasticSearch()
}
// 测试query doc
// test doc query works
func TestQueryDoc(t *testing.T) {
InitConfig()
InitLogs()
@ -80,14 +96,15 @@ func TestQueryDoc(t *testing.T) {
InitMysql()
db.InitELK()
query := elastic.NewTermQuery("content", "title")
query := elastic.NewTermQuery("title", "c")
x := []model.Doc{}
titles, e := db.GetElastic().Query("doc", query, &x, 10, 0)
_, e := db.GetElastic().Query("doc", query, &x, 10, 0)
if nil != e {
log.Print(e.Error())
}
log.Print(x)
log.Print(titles)
for _, v := range x {
log.Print(v.Title, " ", v.ID, " ", v.Type)
}
}
func TestChangeStructFieldThroughStruct(t *testing.T) {

26
utils/reflect.go
View File

@ -70,6 +70,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Int8:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
@ -77,6 +80,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Int16:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
@ -84,6 +90,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Int32:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
@ -91,6 +100,19 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool {
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Int64:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
}
if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 {
return true
}
if typ2 == reflect.Float32 || typ2 == reflect.Float64 {
return true
}
case reflect.Uint:
if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 {
return true
@ -234,9 +256,9 @@ func UnmarshalJson2StructGen(t reflect.Type, maps map[string]interface{}) (inter
for k, v := range maps {
log.Print(k)
if filedName, ok := remap["json"][k]; ok {
// log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
// log.Print(filedName, " ", opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String())
if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) {
// log.Print(opStruct.FieldByName(filedName).CanAddr(), opStruct.FieldByName(filedName).CanSet())
// log.Print(filedName, " can set ", opStruct.FieldByName(filedName).CanSet(), "can addr ", opStruct.FieldByName(filedName).CanAddr())
if opStruct.FieldByName(filedName).CanSet() {
opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type()))
}