diff --git a/db/elasticEngine.go b/db/elasticEngine.go index 1fd4481..7a489d1 100644 --- a/db/elasticEngine.go +++ b/db/elasticEngine.go @@ -61,16 +61,16 @@ func (p *ElkEngine) Delete(query elastic.Query, index string) error { } func (p *ElkEngine) Query(index string, query elastic.Query, v interface{}, - limit int, offset int) ( []string, error) { + limit int, offset int) ([]string, error) { if reflect.ValueOf(v).Kind() != reflect.Ptr { return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Ptr") } if reflect.ValueOf(v).Elem().Kind() != reflect.Slice { - return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice") + return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice") } if reflect.ValueOf(v).Elem().Type().Elem().Kind() != reflect.Struct { - return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct") + return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct") } eletype := reflect.ValueOf(v).Elem().Type().Elem() obj := reflect.ValueOf(v).Elem() @@ -81,7 +81,7 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{}, res, err := p.cli.Search(index).Query(query).Do(context.Background()) if err != nil { print(err) - return nil, err + return nil, err } id := []string{} @@ -103,12 +103,12 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{}, } objAdd = append(objAdd, reflect.ValueOf(obj)) } - return id, nil + return id, nil } else { res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background()) if err != nil { print(err) - return nil, err + return nil, err } id := []string{} for _, vs := range res.Hits.Hits { @@ -123,7 +123,6 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{}, log.Print(e.Error()) } obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj) - log.Print(obj) if nil != e { log.Print(e.Error()) } @@ -131,10 +130,10 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{}, } addOp := reflect.Append(obj, objAdd...) obj.Set(addOp) - return id, nil + return id, nil } } else { - return nil, errors.New(ERROR_PTR) + return nil, errors.New(ERROR_PTR) } } diff --git a/model/blog.go b/model/blog.go index 6bc81d9..627b470 100644 --- a/model/blog.go +++ b/model/blog.go @@ -52,6 +52,7 @@ func GetArticlesType() []ArticleType { } return ret } + /* CreateDoc 新建文档 */ @@ -67,7 +68,7 @@ func CreateDoc(doc Doc) error { DUAL WHERE NOT EXISTS ( SELECT * FROM doc WHERE doc.title = '%s' );`, doc.Title, strings.Replace(doc.Content, "'", "\\'", -1), - doc.Author, doc.Type, doc.IsPublic,time.Now().Format("2006-01-02 15:04:05"), doc.Title) + doc.Author, doc.Type, doc.IsPublic, time.Now().Format("2006-01-02 15:04:05"), doc.Title) _, e := db.GetMysqlClient().Query(sql) if nil != e { log.Print(sql) @@ -76,13 +77,14 @@ func CreateDoc(doc Doc) error { } return nil } + /* UpdateDoc 更新文档 */ func UpdateDoc(doc Doc) error { sql := fmt.Sprintf(`update doc set doc.author = '%s' ,doc.title = '%s',doc.type = '%d',doc.content = '%s' ,doc.update_time = '%s' where doc.id = '%d'; `, doc.Author, doc.Title, doc.Type, - strings.Replace(doc.Content, "'", "\\'", -1), time.Now().Format("2006-01-02 15:04:05"),doc.ID) + strings.Replace(doc.Content, "'", "\\'", -1), time.Now().Format("2006-01-02 15:04:05"), doc.ID) _, e := db.GetMysqlClient().Query(sql) if nil != e { logs.Error(e.Error()) @@ -90,6 +92,7 @@ func UpdateDoc(doc Doc) error { } return nil } + /* DeleteDoc 删除文档 */ @@ -102,6 +105,7 @@ func DeleteDoc(id int64) error { } return nil } + /* AddArticleType 添加文档类型 */ @@ -115,6 +119,7 @@ func AddArticleType(t ArticleType) error { } return nil } + /* UpdateArticleType 更新文档类型 */ @@ -128,6 +133,7 @@ func UpdateArticleType(t ArticleType) error { return nil } + /* DeleteArticleType 删除文档类型 */ diff --git a/model/port.go b/model/port.go index 1573c10..8139b71 100644 --- a/model/port.go +++ b/model/port.go @@ -97,6 +97,49 @@ func InsertDocToElaticSearch(doc Doc) error { return nil } +func CreateIndexFromMysqlTable(tblname string) error { + columns := []Field{} + e := db.GetMysqlClient().Query2("describe "+tblname, &columns) + if nil != e { + logger.Debug(e.Error()) + return e + } + if existed, _ := db.GetElastic().IndexExisted(tblname); existed { + log.Print("index not existed , create " + tblname) + } else { + props := map[string]interface{}{} + mapping := map[string]interface{}{ + "settings": map[string]interface{}{ + "analysis": map[string]interface{}{ + "analyzer": map[string]interface{}{ + "default": map[string]interface{}{ + "type": "smartcn", + }, + }, + }, + }, + "mappings": map[string]interface{}{ + "properties": props, + }, + } + + for _, v := range columns { + props[v.Field] = map[string]string{"type": MysqlToElasticSearchMapping(v.Type, v.Key)} + } + dat, e := json.Marshal(mapping) + if nil != e { + log.Print(e.Error()) + } + e = db.GetElastic().CreateIndex(tblname, string(dat)) + if nil != e { + log.Print(e.Error()) + } + log.Print(string(dat)) + + } + return nil +} + func PortDocumentToElasticsearch(tblname string) error { ret, e := GetAllDocs() if nil != e { diff --git a/test/git_test.go b/test/git_test.go index 7489a9c..1492b24 100644 --- a/test/git_test.go +++ b/test/git_test.go @@ -3,6 +3,7 @@ package test import ( "log" "testing" + "github.com/sergi/go-diff/diffmatchpatch" ) diff --git a/test/portData_test.go b/test/portData_test.go index 83e0aae..07ceab2 100644 --- a/test/portData_test.go +++ b/test/portData_test.go @@ -47,6 +47,9 @@ func InitElasticSearch() { func InitLogs() { logs.Init(config.GetLogConfig().Dir, config.GetLogConfig().File, config.GetLogConfig().Level, config.GetLogConfig().SaveFile) } + +// 从mysql数据自动导出数据到es,从describe 表新建索引和mapping,然后全表导出到es +// port data from mysql to es, it can describe table and create index with mapping ,then port all data to es. func TestPortDocToElastic(t *testing.T) { InitConfig() InitLogs() @@ -73,6 +76,19 @@ func TestReflect(t *testing.T) { log.Print(xx) } +// 测试新建索引,查看mapping是否生效 +// test create new index,and check the mapping works +func TestCreateIndex(t *testing.T) { + InitConfig() + InitLogs() + InitRedisConfig() + InitMysql() + db.InitELK() + InitElasticSearch() +} + +// 测试query doc +// test doc query works func TestQueryDoc(t *testing.T) { InitConfig() InitLogs() @@ -80,14 +96,15 @@ func TestQueryDoc(t *testing.T) { InitMysql() db.InitELK() - query := elastic.NewTermQuery("content", "title") + query := elastic.NewTermQuery("title", "c") x := []model.Doc{} - titles, e := db.GetElastic().Query("doc", query, &x, 10, 0) + _, e := db.GetElastic().Query("doc", query, &x, 10, 0) if nil != e { log.Print(e.Error()) } - log.Print(x) - log.Print(titles) + for _, v := range x { + log.Print(v.Title, " ", v.ID, " ", v.Type) + } } func TestChangeStructFieldThroughStruct(t *testing.T) { diff --git a/utils/reflect.go b/utils/reflect.go index d7f7d2d..c20af90 100644 --- a/utils/reflect.go +++ b/utils/reflect.go @@ -70,6 +70,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool { if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } + if typ2 == reflect.Float32 || typ2 == reflect.Float64 { + return true + } case reflect.Int8: if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true @@ -77,6 +80,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool { if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } + if typ2 == reflect.Float32 || typ2 == reflect.Float64 { + return true + } case reflect.Int16: if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true @@ -84,6 +90,9 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool { if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } + if typ2 == reflect.Float32 || typ2 == reflect.Float64 { + return true + } case reflect.Int32: if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true @@ -91,6 +100,19 @@ func SameKind(typ1 reflect.Kind, typ2 reflect.Kind) bool { if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { return true } + if typ2 == reflect.Float32 || typ2 == reflect.Float64 { + return true + } + case reflect.Int64: + if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { + return true + } + if typ2 == reflect.Uint || typ2 == reflect.Uint16 || typ2 == reflect.Uint32 || typ2 == reflect.Uint64 || typ2 == reflect.Uint8 { + return true + } + if typ2 == reflect.Float32 || typ2 == reflect.Float64 { + return true + } case reflect.Uint: if typ2 == reflect.Int || typ2 == reflect.Int16 || typ2 == reflect.Int32 || typ2 == reflect.Int64 || typ2 == reflect.Int8 { return true @@ -234,9 +256,9 @@ func UnmarshalJson2StructGen(t reflect.Type, maps map[string]interface{}) (inter for k, v := range maps { log.Print(k) if filedName, ok := remap["json"][k]; ok { - // log.Print(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String()) + // log.Print(filedName, " ", opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind().String()) if SameKind(opStruct.FieldByName(filedName).Kind(), reflect.TypeOf(v).Kind()) { - // log.Print(opStruct.FieldByName(filedName).CanAddr(), opStruct.FieldByName(filedName).CanSet()) + // log.Print(filedName, " can set ", opStruct.FieldByName(filedName).CanSet(), "can addr ", opStruct.FieldByName(filedName).CanAddr()) if opStruct.FieldByName(filedName).CanSet() { opStruct.FieldByName(filedName).Set(reflect.ValueOf(v).Convert(opStruct.FieldByName(filedName).Type())) }