调试完成

This commit is contained in:
zcy 2024-05-10 03:42:35 +08:00
parent acbb73d53d
commit 2a9d827b28
9 changed files with 270 additions and 291 deletions

View File

@ -5,6 +5,8 @@ import (
"background/db" "background/db"
"background/logs" "background/logs"
"background/model" "background/model"
"time"
"fmt" "fmt"
"strconv" "strconv"
"strings" "strings"
@ -61,7 +63,12 @@ func InitDocCache() {
} }
for _, v := range articles { for _, v := range articles {
if v.Father != 0 { if v.Father != 0 {
gDocMapCache[int(v.Father)].Children = append(gDocMapCache[int(v.Father)].Children, gDocMapCache[int(v.ID)]) if gDocMapCache[int(v.Father)] != nil {
gDocMapCache[int(v.Father)].Children = append(gDocMapCache[int(v.Father)].Children, gDocMapCache[int(v.ID)])
} else {
log.Print(v.Father)
}
} }
} }
@ -109,6 +116,7 @@ func GetArticlesTree(c *gin.Context) {
rsp.Msg = "OK" rsp.Msg = "OK"
rsp.Status = 0 rsp.Status = 0
} }
func GetArticles(c *gin.Context) { func GetArticles(c *gin.Context) {
type ReqArticles struct { type ReqArticles struct {
Name string `json:"name"` Name string `json:"name"`
@ -495,46 +503,6 @@ func SearchArticleES(c *gin.Context) {
rsp.Status = 0 rsp.Status = 0
} }
func AddArticle(c *gin.Context) {
rsp := RespBase{Msg: "FAIL", Status: 210}
type ReqAddArticle struct {
ID int64 `json:"id"`
Title string `json:"title"`
Content string `json:"content"`
Author string `json:"author"`
Type int64 `json:"type"`
Ispublic int `json:"is_public"`
}
var req ReqAddArticle
defer func() {
c.JSON(200, rsp)
}()
er := c.BindJSON(&req)
if nil != er {
logs.Error(er.Error())
return
}
if req.Title == "" {
rsp.Msg = "title required"
return
}
e := model.CreateDoc(
model.Doc{
Type: int32(req.Type),
Title: req.Title,
Content: req.Content,
Author: req.Author,
IsPublic: int32(req.Ispublic),
},
)
if nil != e {
logs.Error(e.Error())
return
}
rsp.Msg = "OK"
rsp.Status = 0
}
func AddArticleTree(c *gin.Context) { func AddArticleTree(c *gin.Context) {
rsp := RespBase{Msg: "FAIL", Status: 210} rsp := RespBase{Msg: "FAIL", Status: 210}
type ReqAddArticle struct { type ReqAddArticle struct {
@ -560,7 +528,7 @@ func AddArticleTree(c *gin.Context) {
return return
} }
log.Print(req.Level) log.Print(req.Level)
// req.Content = strings.Replace(req.Content, "\\", "\\\\", -1)
e := model.CreateDocTree( e := model.CreateDocTree(
model.DocTree{ model.DocTree{
Type: int32(req.Type), Type: int32(req.Type),
@ -576,13 +544,71 @@ func AddArticleTree(c *gin.Context) {
logs.Error(e.Error()) logs.Error(e.Error())
return return
} }
e = model.InsertCrawDocToElaticSearch(model.DocTree{
Type: int32(req.Type),
Title: req.Title,
Content: req.Content,
Author: req.Author,
IsPublic: int32(req.Ispublic),
Father: int32(req.Father),
Level: int32(req.Level),
CreateTime: time.Now().Format("2006-01-02"),
UpdateTime: time.Now(),
})
if nil != e {
log.Print(e.Error())
}
config.RedisOne().Del("artilcetree")
rsp.Msg = "OK"
rsp.Status = 0
rsp.Data = model.GetIdFromTitle(req.Title)
}
func AddKnowledge(c *gin.Context) {
rsp := RespBase{Msg: "FAIL", Status: 210}
type ReqAddArticle struct {
Title string `json:"title"`
Content string `json:"content"`
Author string `json:"author"`
Type int64 `json:"type"`
Ispublic int `json:"is_public"`
Father int `json:"father"`
Level int `json:"level"`
}
var req ReqAddArticle
defer func() {
c.JSON(200, rsp)
}()
er := c.BindJSON(&req)
if nil != er {
logs.Error(er.Error())
return
}
if req.Title == "" {
rsp.Msg = "title required"
return
}
log.Print(req.Level)
e := model.CreateCrawDoc(
model.DocTree{
Type: int32(req.Type),
Title: req.Title,
Content: req.Content,
Author: req.Author,
IsPublic: int32(req.Ispublic),
Father: int32(req.Father),
Level: int32(req.Level),
},
)
if nil != e {
logs.Error(e.Error())
return
}
rsp.Msg = "OK" rsp.Msg = "OK"
rsp.Status = 0 rsp.Status = 0
rsp.Data = model.GetIdFromTitle(req.Title)
config.RedisOne().Del("artilcetree")
} }
func ArticlesType(c *gin.Context) { func ArticlesType(c *gin.Context) {
rsp := RespBase{Msg: "FAIL", Status: 210} rsp := RespBase{Msg: "FAIL", Status: 210}
defer func() { defer func() {
@ -962,6 +988,7 @@ func GetDocTypeGroup(c *gin.Context) {
rsp.Status = 0 rsp.Status = 0
rsp.Msg = "OK" rsp.Msg = "OK"
} }
func GetDoGroupcType(c *gin.Context) { func GetDoGroupcType(c *gin.Context) {
type DocType struct { type DocType struct {
Id int32 `json:"id"` Id int32 `json:"id"`

View File

@ -7,7 +7,9 @@ import (
"fmt" "fmt"
"image" "image"
"image/color" "image/color"
"image/gif"
"image/jpeg" "image/jpeg"
"io"
"io/ioutil" "io/ioutil"
"log" "log"
"math" "math"
@ -24,8 +26,12 @@ import (
// "github.com/nfnt/resize" // "github.com/nfnt/resize"
// "image/jpeg" // "image/jpeg"
"image/gif"
// "strings" // "strings"
"bufio"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
) )
type FileController struct { type FileController struct {
@ -68,6 +74,49 @@ func (this *FileController) OnUploadDoc(c *gin.Context) {
} }
func DownloadImg(imgUrl, filePath string) {
res, err := http.Get(imgUrl)
if err != nil {
fmt.Println("A error occurred!")
return
}
defer res.Body.Close()
// 获得get请求响应的reader对象
reader := bufio.NewReaderSize(res.Body, 32*1024)
file, err := os.Create(filePath)
if err != nil {
panic(err)
}
// 获得文件的writer对象
writer := bufio.NewWriter(file)
written, _ := io.Copy(writer, reader)
fmt.Printf("Total length: %d", written)
file.Close()
}
func (this *FileController) OnUpaloadPic(c *gin.Context) {
uid, e := uuid.NewV1()
if nil != e {
log.Print(e.Error())
return
}
type Url struct {
Url string `json:"url"`
}
var url Url
e = c.BindJSON(&url)
if nil != e {
log.Print(e.Error())
return
}
DownloadImg(url.Url, "image/"+uid.String()+".png")
c.JSON(200, map[string]interface{}{"url": uid.String() + ".png"})
}
func (this *FileController) OnUpload(c *gin.Context) { func (this *FileController) OnUpload(c *gin.Context) {
uid, e := uuid.NewV1() uid, e := uuid.NewV1()
if nil != e { if nil != e {

View File

@ -26,7 +26,7 @@ type ElkEngine struct {
func (p *ElkEngine) Create(index string, types string, id string, data interface{}) error { func (p *ElkEngine) Create(index string, types string, id string, data interface{}) error {
if nil != p { if nil != p {
if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct) { if (reflect.TypeOf(data).Kind() == reflect.String) || (reflect.TypeOf(data).Kind() == reflect.Struct) {
resp, err := p.cli.Index(). _, err := p.cli.Index().
Index(index). Index(index).
BodyJson(data). BodyJson(data).
Do(context.Background()) Do(context.Background())
@ -34,7 +34,6 @@ func (p *ElkEngine) Create(index string, types string, id string, data interface
log.Print("create error", err) log.Print("create error", err)
return err return err
} }
log.Print(resp)
} else { } else {
log.Print(reflect.TypeOf(data).Kind()) log.Print(reflect.TypeOf(data).Kind())
return errors.New(INPUT_TYPE_ERROR) return errors.New(INPUT_TYPE_ERROR)
@ -162,7 +161,6 @@ func (p *ElkEngine) Query(index string, query elastic.Query, v interface{},
print(err) print(err)
return nil, err return nil, err
} }
id := []string{} id := []string{}
for _, vs := range res.Hits.Hits { for _, vs := range res.Hits.Hits {
id = append(id, vs.Id) id = append(id, vs.Id)

View File

@ -121,14 +121,13 @@ func InitELK() {
elastic.SetBasicAuth(elkconf.User, elkconf.Password), elastic.SetBasicAuth(elkconf.User, elkconf.Password),
) )
log.Print("init elk cli is ",gElkEngine.cli,elkconf) log.Print("init elk cli is ", gElkEngine.cli, elkconf)
if nil != e { if nil != e {
logs.Error(e.Error()) logs.Error(e.Error())
gElkEngine.cli = nil gElkEngine.cli = nil
} }
} }
func initMysql(mysqlconf *config.MysqlConfig) *sql.DB { func initMysql(mysqlconf *config.MysqlConfig) *sql.DB {
cnn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s?charset=utf8", mysqlconf.UserName, mysqlconf.Password, cnn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s?charset=utf8", mysqlconf.UserName, mysqlconf.Password,
mysqlconf.Addr, mysqlconf.Db) mysqlconf.Addr, mysqlconf.Db)

View File

@ -39,7 +39,6 @@ func CORSMiddleware(c *gin.Context) {
c.Writer.Header().Set("Access-Control-Allow-Origin", "https://testingcloud.club") c.Writer.Header().Set("Access-Control-Allow-Origin", "https://testingcloud.club")
} else { } else {
c.Writer.Header().Set("Access-Control-Allow-Origin", "https://testingcloud.club") c.Writer.Header().Set("Access-Control-Allow-Origin", "https://testingcloud.club")
} }
c.Writer.Header().Set("Access-Control-Max-Age", "86400") c.Writer.Header().Set("Access-Control-Max-Age", "86400")
c.Writer.Header().Set("Access-Control-Allow-Headers", c.Writer.Header().Set("Access-Control-Allow-Headers",
@ -163,10 +162,8 @@ func main() {
api.POST("/articles", controller.GetArticles) // 获取所有文章 api.POST("/articles", controller.GetArticles) // 获取所有文章
api.GET("/articles_tree", controller.GetArticlesTree) // 获取所有文章 api.GET("/articles_tree", controller.GetArticlesTree) // 获取所有文章
api.PUT("/article", controller.AddArticle) // 添加文章
api.PUT("/article_file", controller.AddArticle) // 添加文章
api.PUT("/article_tree", controller.AddArticleTree) // 添加文章 api.PUT("/article_tree", controller.AddArticleTree) // 添加文章
api.PUT("/knowledge", controller.AddKnowledge) // 添加文章
api.PUT("/article_tree_history", controller.AddArticleHistory) // 添加文章 api.PUT("/article_tree_history", controller.AddArticleHistory) // 添加文章
api.PUT("/article_search", controller.SearchArticles) // 添加文章 api.PUT("/article_search", controller.SearchArticles) // 添加文章
@ -184,6 +181,7 @@ func main() {
api.DELETE("/article_tree/:id", controller.DeleteArticleTree) ////删除文章 api.DELETE("/article_tree/:id", controller.DeleteArticleTree) ////删除文章
api.GET("/article_history", controller.GetArticleHistory) // 历史 api.GET("/article_history", controller.GetArticleHistory) // 历史
api.POST("/download_pic", fileController.OnUpaloadPic) // 获取类所在的组
api.POST("/image_upload", fileController.OnUpload) // 上传图片,如果图片太大自动裁减为原来的一半 api.POST("/image_upload", fileController.OnUpload) // 上传图片,如果图片太大自动裁减为原来的一半
api.POST("/image_upload_origin", fileController.OnUploadOrigin) // 上传图片 api.POST("/image_upload_origin", fileController.OnUploadOrigin) // 上传图片
api.GET("/image_download/:file", fileController.OnDownLoad) // 下载图片 api.GET("/image_download/:file", fileController.OnDownLoad) // 下载图片

View File

@ -11,6 +11,25 @@ import (
"qiniupkg.com/x/log.v7" "qiniupkg.com/x/log.v7"
) )
type CrawDoc struct {
Id int32 `json:"id" gorm:"column:id" sql:"id"`
Title string `json:"title" gorm:"column:title" sql:"title"`
Type int32 `json:"type" gorm:"column:type" sql:"type"`
Content []byte `json:"content" gorm:"column:content" sql:"content"`
Author string `json:"author" gorm:"column:author" sql:"author"`
CreateTime time.Time `json:"create_time" gorm:"column:create_time" sql:"create_time"`
UpdateTime time.Time `json:"update_time" gorm:"column:update_time" sql:"update_time"`
DeleteTime time.Time `json:"delete_time" gorm:"column:delete_time" sql:"delete_time"`
Version int32 `json:"version" gorm:"column:version" sql:"version"`
IsPublic int32 `json:"is_public" gorm:"column:is_public" sql:"is_public"`
Deleted int32 `json:"deleted" gorm:"column:deleted" sql:"deleted"`
OriginUrl string `json:"origin_url" gorm:"column:origin_url" sql:"origin_url"`
EsId string `json:"es_id" gorm:"column:es_id" sql:"es_id"`
DeletedAt time.Time `json:"deleted_at" gorm:"column:deleted_at" sql:"deleted_at"`
Father int32 `json:"father" gorm:"column:father" sql:"father"`
Level int32 `json:"level" gorm:"column:level" sql:"level"`
}
type Doc struct { type Doc struct {
ID int64 `json:"id" gorm:"column:id" sql:"id"` ID int64 `json:"id" gorm:"column:id" sql:"id"`
Title string `json:"title" gorm:"column:title" sql:"title"` Title string `json:"title" gorm:"column:title" sql:"title"`
@ -36,7 +55,7 @@ type DocTree struct {
Type int32 `json:"type" gorm:"column:type" sql:"type"` Type int32 `json:"type" gorm:"column:type" sql:"type"`
Content string `json:"content" gorm:"column:content" sql:"content"` Content string `json:"content" gorm:"column:content" sql:"content"`
Author string `json:"author" gorm:"column:author" sql:"author"` Author string `json:"author" gorm:"column:author" sql:"author"`
CreateTime time.Time `json:"create_time" gorm:"column:create_time" sql:"create_time"` CreateTime string `json:"create_time" gorm:"column:create_time" sql:"create_time"`
UpdateTime time.Time `json:"update_time" gorm:"column:update_time" sql:"update_time"` UpdateTime time.Time `json:"update_time" gorm:"column:update_time" sql:"update_time"`
DeleteTime time.Time `json:"delete_time" gorm:"column:delete_time" sql:"delete_time"` DeleteTime time.Time `json:"delete_time" gorm:"column:delete_time" sql:"delete_time"`
Version int32 `json:"version" gorm:"column:version" sql:"version"` Version int32 `json:"version" gorm:"column:version" sql:"version"`
@ -54,6 +73,11 @@ func (DocTree) TableName() string {
return "doc_copy1" return "doc_copy1"
} }
// 设置User的表名为`profiles`
func (CrawDoc) TableName() string {
return "craw_doc"
}
type DocType struct { type DocType struct {
Id int32 `json:"id" gorm:"column:id" sql:"id"` Id int32 `json:"id" gorm:"column:id" sql:"id"`
TypeName string `json:"type_name" gorm:"column:type_name" sql:"type_name"` TypeName string `json:"type_name" gorm:"column:type_name" sql:"type_name"`
@ -100,31 +124,6 @@ func GetArticlesType() []ArticleType {
return ret return ret
} }
/*
CreateDoc 新建文档
*/
func CreateDoc(doc Doc) error {
sql := fmt.Sprintf(`INSERT INTO doc ( doc.title, doc.content, doc.author, doc.type,doc.is_public,doc.create_time) SELECT
'%s',
'%s',
'%s',
%d ,
%d,
'%s'
FROM
DUAL
WHERE
NOT EXISTS ( SELECT * FROM doc WHERE doc.title = '%s' );`, doc.Title, strings.Replace(doc.Content, "'", "\\'", -1),
doc.Author, doc.Type, doc.IsPublic, time.Now().Format("2006-01-02 15:04:05"), doc.Title)
_, e := db.GetMysqlClient().Query(sql)
if nil != e {
log.Print(sql)
logs.Error(e.Error())
return e
}
return nil
}
/* /*
CreateDoc 新建文档 CreateDoc 新建文档
*/ */
@ -154,6 +153,34 @@ func CreateDocTree(doc DocTree) error {
return nil return nil
} }
/*
CreateDoc 新建文档
*/
func CreateCrawDoc(doc DocTree) error {
sql := fmt.Sprintf(`INSERT INTO craw_doc ( craw_doc.title, craw_doc.content,
craw_doc.author, craw_doc.type,craw_doc.is_public,craw_doc.create_time,father,level) SELECT
'%s',
'%s',
'%s',
%d ,
%d,
'%s',
%d,
%d
FROM
DUAL
WHERE
NOT EXISTS ( SELECT * FROM craw_doc WHERE craw_doc.title = '%s' );`, doc.Title, strings.Replace(doc.Content, "'", "\\'", -1),
doc.Author, doc.Type, doc.IsPublic, time.Now().Format("2006-01-02 15:04:05"), doc.Father, doc.Level, doc.Title)
_, e := db.GetMysqlClient().Query(sql)
log.Print(sql)
if nil != e {
log.Print(sql)
logs.Error(e.Error())
return e
}
return nil
}
func GetIdFromTitle(title string) int { func GetIdFromTitle(title string) int {
motors := []DocTree{} motors := []DocTree{}
@ -286,9 +313,19 @@ func DeleteArticleType(id int32) error {
return nil return nil
} }
func GetAllDocs() ([]DocTree, error) { func GetAllDocs(tbl string) ([]DocTree, error) {
ret := []DocTree{} ret := []DocTree{}
sql := fmt.Sprintf("select * from doc_copy1") sql := fmt.Sprintf("select * from " + tbl)
e := db.GetMysqlClient().Query2(sql, &ret)
if nil != e {
logs.Error(e.Error())
return nil, e
}
return ret, nil
}
func GetAllCraw() ([]DocTree, error) {
ret := []DocTree{}
sql := fmt.Sprintf("select * from craw_doc")
e := db.GetMysqlClient().Query2(sql, &ret) e := db.GetMysqlClient().Query2(sql, &ret)
if nil != e { if nil != e {
logs.Error(e.Error()) logs.Error(e.Error())
@ -296,7 +333,6 @@ func GetAllDocs() ([]DocTree, error) {
} }
return ret, nil return ret, nil
} }
func GetAllGroup() ([]DocGroup, error) { func GetAllGroup() ([]DocGroup, error) {
ret := []DocGroup{} ret := []DocGroup{}
sql := fmt.Sprintf("select * from doc_group") sql := fmt.Sprintf("select * from doc_group")

View File

@ -91,6 +91,23 @@ func QueryHardwares(query elastic.Query, limit int, offset int) ([]Hardware, err
return ret, nil return ret, nil
} }
func QueryDoc(query elastic.Query, limit int, offset int) ([]DocTree, error) {
ret := []DocTree{}
_, e := db.GetElastic().Query("doc", query, &ret, limit, offset)
if nil != e {
return nil, e
}
return ret, nil
}
func QueryKnowledge(query elastic.Query, limit int, offset int) ([]DocTree, error) {
ret := []DocTree{}
_, e := db.GetElastic().Query("doc", query, &ret, limit, offset)
if nil != e {
return nil, e
}
return ret, nil
}
func DeleteHardware(name string) error { func DeleteHardware(name string) error {
query := elastic.NewTermQuery("name", name) query := elastic.NewTermQuery("name", name)
err := db.GetElastic().Delete(query, "hardware_data") err := db.GetElastic().Delete(query, "hardware_data")

View File

@ -5,7 +5,6 @@ import (
"strings" "strings"
"ubntgo/logger" "ubntgo/logger"
"github.com/go-openapi/errors"
json "github.com/json-iterator/go" json "github.com/json-iterator/go"
"gopkg.in/olivere/elastic.v3" "gopkg.in/olivere/elastic.v3"
"qiniupkg.com/x/log.v7" "qiniupkg.com/x/log.v7"
@ -78,24 +77,47 @@ func QueryDocument(query elastic.Query, limit int, offset int) ([]Hardware, erro
return ret, nil return ret, nil
} }
func InsertDocToElaticSearch(doc DocTree) error { func InsertDocToElaticSearch(doc DocTree) error {
matchPhraseQuery := elastic.NewMatchQuery("title", doc.Title) matchPhraseQuery := elastic.NewTermQuery("title.keyword", doc.Title)
existedHardware, e := QueryHardwares(matchPhraseQuery, 10, 0) existedHardware, e := QueryDoc(matchPhraseQuery, 10, 0)
log.Print(e, existedHardware)
for _, v := range existedHardware {
if v.Name == doc.Title {
log.Print(v.ID)
return errors.New(200, "existed title")
}
}
e = db.GetElastic().Create("doc", "0", "", doc)
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
return e return e
} }
if len(existedHardware) == 0 {
log.Print("doc created ", doc.Title)
e = db.GetElastic().Create("doc", "0", "", doc)
if nil != e {
log.Print(e.Error())
return e
}
return nil
} else {
log.Print("doc existed ", doc.Title)
}
return nil return nil
} }
func InsertCrawDocToElaticSearch(doc DocTree) error {
matchPhraseQuery := elastic.NewTermQuery("title.keyword", doc.Title)
existedHardware, e := QueryDoc(matchPhraseQuery, 10, 0)
if nil != e {
log.Print(e.Error())
return e
}
if len(existedHardware) == 0 {
log.Print("doc created ", doc.Title)
e = db.GetElastic().Create("doc", "0", "", doc)
if nil != e {
log.Print(e.Error())
return e
}
return nil
} else {
log.Print("doc existed ", doc.Title)
}
return nil
}
func CreateIndexFromMysqlTable(tblname string) error { func CreateIndexFromMysqlTable(tblname string) error {
columns := []Field{} columns := []Field{}
e := db.GetMysqlClient().Query2("describe "+tblname, &columns) e := db.GetMysqlClient().Query2("describe "+tblname, &columns)
@ -138,8 +160,8 @@ func CreateIndexFromMysqlTable(tblname string) error {
return nil return nil
} }
func PortDocumentToElasticsearch(tblname string) error { func PortDocumentToElasticsearch(tblname string, indexname string) error {
ret, e := GetAllDocs() ret, e := GetAllDocs(tblname)
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
@ -149,7 +171,7 @@ func PortDocumentToElasticsearch(tblname string) error {
logger.Debug(e.Error()) logger.Debug(e.Error())
return e return e
} }
if existed, _ := db.GetElastic().IndexExisted(tblname); !existed { if existed, _ := db.GetElastic().IndexExisted(indexname); !existed {
props := map[string]interface{}{} props := map[string]interface{}{}
mapping := map[string]interface{}{ mapping := map[string]interface{}{
"settings": map[string]interface{}{ "settings": map[string]interface{}{
@ -168,20 +190,36 @@ func PortDocumentToElasticsearch(tblname string) error {
} }
for _, v := range columns { for _, v := range columns {
props[v.Field] = map[string]string{"type": MysqlToElasticSearchMapping(v.Type, v.Key)} props[v.Field] = map[string]interface{}{"type": MysqlToElasticSearchMapping(v.Type, v.Key)}
if MysqlToElasticSearchMapping(v.Type, v.Key) == "text" {
props[v.Field].(map[string]interface{})["fields"] = map[string]interface{}{
"keyword": map[string]interface{}{
"type": "keyword",
},
}
}
} }
dat, e := json.Marshal(mapping) dat, e := json.Marshal(mapping)
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
log.Print(string(dat)) log.Print(string(dat))
e = db.GetElastic().CreateIndex(tblname, string(dat)) e = db.GetElastic().CreateIndex(indexname, string(dat))
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
log.Print(string(dat)) log.Print(string(dat))
for _, v := range ret { for _, v := range ret {
e := InsertDocToElaticSearch(v) e := InsertCrawDocToElaticSearch(v)
if nil != e {
log.Print(e.Error())
}
}
} else {
for _, v := range ret {
e := InsertCrawDocToElaticSearch(v)
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }

View File

@ -59,24 +59,22 @@ func TestPortDocToElastic(t *testing.T) {
InitRedisConfig() InitRedisConfig()
InitMysql() InitMysql()
db.InitELK() db.InitELK()
e := model.PortDocumentToElasticsearch("doc_copy1") e := model.PortDocumentToElasticsearch("doc_copy1", "doc")
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
} }
func TestPortCrawToElastic(t *testing.T) {
func TestPortDocTreeToElastic(t *testing.T) {
InitConfig() InitConfig()
InitLogs() InitLogs()
InitRedisConfig() InitRedisConfig()
InitMysql() InitMysql()
db.InitELK() db.InitELK()
e := model.PortDocumentToElasticsearch("doc_copy1") e := model.PortDocumentToElasticsearch("craw_doc", "craw_doc")
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
} }
func TestReflect(t *testing.T) { func TestReflect(t *testing.T) {
type XX struct { type XX struct {
A int16 `json:"bb"` A int16 `json:"bb"`
@ -111,12 +109,13 @@ func TestQueryDoc(t *testing.T) {
InitMysql() InitMysql()
db.InitELK() db.InitELK()
query := elastic.NewTermQuery("title", "c") query := elastic.NewTermQuery("title.keyword", "QT 下tableview的坑")
x := []model.Doc{} x := []model.Doc{}
_, e := db.GetElastic().Query("doc", query, &x, 10, 0) _, e := db.GetElastic().Query("doc", query, &x, 10, 0)
if nil != e { if nil != e {
log.Print(e.Error()) log.Print(e.Error())
} }
t.Log(x)
for _, v := range x { for _, v := range x {
t.Logf(v.Title, " ", v.ID, " ", v.Type) t.Logf(v.Title, " ", v.ID, " ", v.Type)
} }
@ -140,185 +139,3 @@ func TestChangeStructFieldThroughStruct(t *testing.T) {
fmt.Println(cap(e0)) fmt.Println(cap(e0))
fmt.Println(cap(arr1)) fmt.Println(cap(arr1))
} }
func TestPortDocVersion(t *testing.T) {
InitConfig()
InitLogs()
InitRedisConfig()
InitMysql()
docs := []model.Doc{}
doc_groups := []model.DocGroup{}
doc_types := []model.DocType{}
e := db.GetMysqlClient().Query2("select * from doc_group", &doc_groups)
if nil != e {
log.Print(e.Error())
}
// for k,v := range doc_groups{
// log.Print(k,v)
// e := db.GetMysqlClient().Query2("select * from doc_group",&doc_groups)
// if(nil != e){
// log.Print(e.Error())
// }
// str := fmt.Sprintf("insert into doc_copy1 (title,type,content,author,create_time,version,is_public, deleted,origin_url,es_id,father,level) values ('%s',0,'','admin','%s', 1,1,0,'','',0,0)",
// v.Name,time.Now().Format("2006-01-02 15:04:05"))
// log.Print(str)
// _,e = db.GetMysqlClient().Query(str)
// if nil != e{
// log.Print(e.Error())
// }
// }
// e = db.GetMysqlClient().Query2("select * from doc_type",&doc_types)
// if(nil != e){
// log.Print(e.Error())
// }
// for k,v := range doc_types{
// log.Print(k,v)
// doc_group := []model.DocGroup{}
// e = db.GetMysqlClient().Query2(fmt.Sprintf("select * from doc_group where doc_group.int = '%d'",v.Group),&doc_group)
// if nil != e{
// log.Print(e.Error())
// }
// log.Print(doc_group[0])
// doc := []model.Doc{}
// e = db.GetMysqlClient().Query2(fmt.Sprintf("select * from doc_copy1 where doc_copy1.title = '%s' and doc_copy1.father=0",doc_group[0].Name),&doc)
// if nil != e{
// log.Print(e.Error())
// }
// log.Print(doc)
// str := fmt.Sprintf("insert into doc_copy1 (title,type,content,author,create_time,version,is_public, deleted,origin_url,es_id,father,level) values ('%s',0,'','admin','%s', 1,1,0,'','',%d,1)",
// v.TypeName,time.Now().Format("2006-01-02 15:04:05"),doc[0].ID)
// log.Print(str)
// _,e = db.GetMysqlClient().Query(str)
// if nil != e{
// log.Print(e.Error())
// }
// }
e = db.GetMysqlClient().Query2("select id,title,type from doc_copy1 where doc_copy1.father is NULL ", &docs)
if nil != e {
log.Print(e.Error())
}
for k, v := range docs {
log.Print(k, v)
e = db.GetMysqlClient().Query2(fmt.Sprintf("select * from doc_type where doc_type.id = %d ", v.Type), &doc_types)
if nil != e {
log.Print(e.Error())
}
log.Print(doc_types)
if len(doc_types) > 0 {
docsfortype := []model.Doc{}
e = db.GetMysqlClient().Query2(fmt.Sprintf("select * from doc_copy1 where doc_copy1.title = '%s' and doc_copy1.level = 1",
doc_types[0].TypeName), &docsfortype)
if nil != e {
log.Print(e.Error())
}
log.Print(docsfortype)
if len(docsfortype) > 0 {
str := fmt.Sprintf("update doc_copy1 set father = %d ,level = %d where doc_copy1.id = %d",
docsfortype[0].ID, docsfortype[0].Level+1, (v.ID))
log.Print(str)
_, e = db.GetMysqlClient().Query(str)
if nil != e {
log.Printf(e.Error())
}
}
}
}
}
func TestPortDocTree(t *testing.T) {
InitConfig()
InitLogs()
InitRedisConfig()
InitMysql()
docs := []model.Doc{}
doc_groups := []model.DocGroup{}
doc_types := []model.DocType{}
e := db.GetMysqlClient().Query2("select * from doc_group", &doc_groups)
if nil != e {
log.Print(e.Error())
}
// for k,v := range doc_groups{
// log.Print(k,v)
// e := db.GetMysqlClient().Query2("select * from doc_group",&doc_groups)
// if(nil != e){
// log.Print(e.Error())
// }
// str := fmt.Sprintf("insert into doc_copy1 (title,type,content,author,create_time,version,is_public, deleted,origin_url,es_id,father,level) values ('%s',0,'','admin','%s', 1,1,0,'','',0,0)",
// v.Name,time.Now().Format("2006-01-02 15:04:05"))
// log.Print(str)
// _,e = db.GetMysqlClient().Query(str)
// if nil != e{
// log.Print(e.Error())
// }
// }
// e = db.GetMysqlClient().Query2("select * from doc_type",&doc_types)
// if(nil != e){
// log.Print(e.Error())
// }
// for k,v := range doc_types{
// log.Print(k,v)
// doc_group := []model.DocGroup{}
// e = db.GetMysqlClient().Query2(fmt.Sprintf("select * from doc_group where doc_group.int = '%d'",v.Group),&doc_group)
// if nil != e{
// log.Print(e.Error())
// }
// log.Print(doc_group[0])
// doc := []model.Doc{}
// e = db.GetMysqlClient().Query2(fmt.Sprintf("select * from doc_copy1 where doc_copy1.title = '%s' and doc_copy1.father=0",doc_group[0].Name),&doc)
// if nil != e{
// log.Print(e.Error())
// }
// log.Print(doc)
// str := fmt.Sprintf("insert into doc_copy1 (title,type,content,author,create_time,version,is_public, deleted,origin_url,es_id,father,level) values ('%s',0,'','admin','%s', 1,1,0,'','',%d,1)",
// v.TypeName,time.Now().Format("2006-01-02 15:04:05"),doc[0].ID)
// log.Print(str)
// _,e = db.GetMysqlClient().Query(str)
// if nil != e{
// log.Print(e.Error())
// }
// }
e = db.GetMysqlClient().Query2("select id,title,type from doc_copy1 where doc_copy1.father is NULL ", &docs)
if nil != e {
log.Print(e.Error())
}
for k, v := range docs {
log.Print(k, v)
e = db.GetMysqlClient().Query2(fmt.Sprintf("select * from doc_type where doc_type.id = %d ", v.Type), &doc_types)
if nil != e {
log.Print(e.Error())
}
log.Print(doc_types)
if len(doc_types) > 0 {
docsfortype := []model.Doc{}
e = db.GetMysqlClient().Query2(fmt.Sprintf("select * from doc_copy1 where doc_copy1.title = '%s' and doc_copy1.level = 1",
doc_types[0].TypeName), &docsfortype)
if nil != e {
log.Print(e.Error())
}
log.Print(docsfortype)
if len(docsfortype) > 0 {
str := fmt.Sprintf("update doc_copy1 set father = %d ,level = %d where doc_copy1.id = %d",
docsfortype[0].ID, docsfortype[0].Level+1, (v.ID))
log.Print(str)
_, e = db.GetMysqlClient().Query(str)
if nil != e {
log.Printf(e.Error())
}
}
}
}
}