diff --git a/README.MD b/README.MD index f368839..820f7cb 100644 --- a/README.MD +++ b/README.MD @@ -18,13 +18,13 @@ STVM(truck of Virtual memory table)是一个开源的使用ANSI C语言编 ***使用须知*** * 表空间大小在创建时已经确定, 一旦创建不能随意修改大小。 * 多机集群无法使用事务。 -* 数据无法持久化(宕机情况)。 +* 数据无法持久化(宕机情况), 可以定时利用接口备份。 * 索引长度限制(默认64字节),修改需编译项目。 * 启动后不能修改表字段,但可以修改字段别名 建议使用场景:***数据库缓存、数据缓存、事件驱动*** -不定时更新 https://my.oschina.net/deffpuzzl/blog/1627626 +不定时更新 https://my.oschina.net/deffpuzzl/blog/1627626 ## 1、编译 **编译项目** @@ -46,17 +46,20 @@ cd src目录 **./make** * **TVMCFG=$(TVMDBD)/.tvm.run** 启动参数 使用方法: -如果不使用多机集群,无需配置域文件,缺省本地启动。 +如果不使用多机集群,无需配置域文件,缺省本地启动(该版本对集群支持较弱, 时间有限,稍后更新,请悉知)。 -启动系统 +启动系统 > stvm -w -停止系统 -> stvm -s +停止系统 +> stvm -s STVM也提供一个类型sqlpuls类型简单工具。 -进入SQL界面, 该工具主要用来运维调试使用 -> stvm SQL +进入SQL界面, 该工具主要用来运维调试使用,其中showmode显示模式ROW|COLUMN,showsize单次显示记录数,也可用封装该指令的脚本**msql**进入 +> stvm SQL --showmode=row --showsize=8 + +批量执行M-SQL脚本**exsql**,也可直接执行 +>stvm SQL --msql=msql.bat 域的维护 > stvm DOM @@ -67,7 +70,7 @@ STVM也提供一个类型sqlpuls类型简单工具。 配置域完成后,可以配置编译到$TVMCFG下 > stvm -c stvm.conf -然后在stvm -w启动即可 +然后在 **stvm -w** 启动即可 建议以单机模式64位下运行,将性能发挥在极致。 @@ -116,6 +119,12 @@ STVM也提供一个类型sqlpuls类型简单工具。 >* 1、新增queue网络同步异步接口 >* 1、新增将表中长时间无用数据导出备份接口 +队列的创建与表类似: +队列不支持创建索引,队列创建有2种类型,基于表队列和字符串队列,详见API。 +创建队列语法示例: +在M-SQL中执行 create que_user_info.def + + **下本版本:** * 0、新增分布式锁 * 1、新增多机资源共享方式。 @@ -123,13 +132,3 @@ STVM也提供一个类型sqlpuls类型简单工具。 **不定时更新** - - - - ---- -#### *常见错误集合* #### -``` -Q、failed to boot TVM, Invalid parameter or sem has disappeared -A:echo "500 X X X" > /proc/sys/kernel/sem -``` diff --git a/demo/popup.c b/demo/popup.c index c1349cf..b25baa4 100644 --- a/demo/popup.c +++ b/demo/popup.c @@ -29,7 +29,7 @@ int main(int argc, char *argv[]) lTime = time(NULL); queuenull(pstSavm, sizeof(dbUser), QUEUE_USER_INFO); - if(RC_SUCC != lPopup(pstSavm, 3, 5, &lRows, (void **)&pstUser)) + if(RC_SUCC != lPopup(pstSavm, 3, 0, &lRows, (void **)&pstUser)) { fprintf(stderr, "Pop error:(%u)(%s), time:%ld, effect:%d\n", pstSavm->m_lErrno, sGetTError(pstSavm->m_lErrno), time(NULL) - lTime, pstSavm->m_lEffect); diff --git a/include/tmain.h b/include/tmain.h index 85a16c1..0a4d4b3 100644 --- a/include/tmain.h +++ b/include/tmain.h @@ -23,7 +23,6 @@ #define _HHH_TVM_DOMAIN_HHH__ #include "tvm.h" - typedef struct epoll_event epollevt; typedef long (*FUNCEXEC)(SATvm *pstSavm, void *arg); diff --git a/include/tstr.h b/include/tstr.h index 09c8bd7..f61951f 100644 --- a/include/tstr.h +++ b/include/tstr.h @@ -62,6 +62,8 @@ #include #include #include +#define _GNU_SOURCE +#include typedef short int sint; typedef unsigned int uint; diff --git a/include/tvm.h b/include/tvm.h index 10e3b91..aab2aeb 100644 --- a/include/tvm.h +++ b/include/tvm.h @@ -328,9 +328,9 @@ typedef long CREATE; #define IDX_FIELD(t, f, a) if(RC_SUCC != lAddIdxField(tbl, type, FPOS(t, f), FLEN(t, f), a, #f)) \ return RC_FAIL; #define FIELD(t, f, a) if(RC_SUCC != lSetTableIdx(tbl, FPOS(t, f), FLEN(t, f), #f, a, CHK_SELECT)) \ - return RC_FAIL; + return RC_FAIL; #define FIELU(t, f, a) if(RC_SUCC != lSetTableIdx(tbl, FPOS(t, f), FLEN(t, f), #f, a, IDX_SELECT)) \ - return RC_FAIL; + return RC_FAIL; #define FIELR(t, f, a) if(RC_SUCC != lSetTableIdx(tbl, FPOS(t, f), FLEN(t, f), #f, a, RCD_SELECT)) \ return RC_FAIL; #define FINISH return RC_SUCC; @@ -339,6 +339,15 @@ typedef long CREATE; /************************************************************************************************* Field assignment *************************************************************************************************/ +#define conditvoid(p,v,l,t) do{ \ + p->stCond.uFldcmp = 0; \ + p->stUpdt.uFldcmp = 0; \ + p->lFind = 0; \ + p->tblName = t; \ + p->lSize = l; \ + p->pstVoid = (void *)&(v); \ + }while(0); + #define conditbind(p,v,t) do{ \ p->stCond.uFldcmp = 0; \ p->stUpdt.uFldcmp = 0; \ @@ -367,6 +376,12 @@ typedef long CREATE; p->pstVoid = NULL; \ }while(0); +#define queuevoid(p,v,l,t) do{ \ + p->lSize = l; \ + p->tblName = t; \ + p->pstVoid = (void *)&v; \ + }while(0); + #define queueinit(p,v,t) do{ \ p->lSize = sizeof(v); \ p->tblName = t; \ diff --git a/src/func.c b/src/func.c index 5544365..fc55903 100644 --- a/src/func.c +++ b/src/func.c @@ -19,12 +19,14 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#define _GNU_SOURCE #include "tvm.h" /************************************************************************************************ function ************************************************************************************************/ extern void vCondInsInit(FdCond *pstCond, TABLE t); +extern long _lDumpTable(SATvm *pstSavm, TABLE t, char *pszFile); /************************************************************************************************* description:dump the unused @@ -100,6 +102,96 @@ long lUnuseDump(SATvm *pstSavm, TABLE t) return RC_SUCC; } +/************************************************************************************************* + description:exit and backup tables + parameters: + pstSavm --stvm handle + return: + RC_SUCC --success + RC_FAIL --failure + *************************************************************************************************/ +long lBackupTables(SATvm *pstSavm) +{ + size_t lOut = 0, i; + char szFile[512]; + TIndex *pstIndex = NULL; + + memset(szFile, 0, sizeof(szFile)); + if(RC_SUCC != lExportTable(SYS_TVM_INDEX, &lOut, (void *)&pstIndex)) + return RC_FAIL; + + snprintf(szFile, sizeof(szFile), "%s/backup", getenv("TVMDBD")); + if(0 != mkdir(szFile, S_IRWXU | S_IRGRP)) + { + if(EEXIST != errno) + { + vRedeError(pstSavm->m_lErrno = 127, strerror(errno)); + return RC_FAIL; + } + } + + for(i = 0; i < lOut; i ++) + { + if((TYPE_SYSTEM == pstIndex[i].m_lType || TYPE_INCORE == pstIndex[i].m_lType) && + SYS_TVM_SEQUE != pstIndex[i].m_table) + continue; + + memset(szFile, 0, sizeof(szFile)); + snprintf(szFile, sizeof(szFile), "%s/backup/%d.sdb", getenv("TVMDBD"), + pstIndex[i].m_table); + _lDumpTable(pstSavm, pstIndex[i].m_table, szFile); + } + + TFree(pstIndex); + + return RC_SUCC; +} + +/************************************************************************************************* + description:boot and restore tables + parameters: + pstSavm --stvm handle + return: + RC_SUCC --success + RC_FAIL --failure + *************************************************************************************************/ +long lRestoreTables(SATvm *pstSavm) +{ + DIR *dir; + struct dirent *pr; + char szPath[512], szFile[128]; + + memset(szPath, 0, sizeof(szPath)); + memset(szFile, 0, sizeof(szFile)); + snprintf(szPath, sizeof(szPath), "%s/backup", getenv("TVMDBD")); + if ((NULL == (dir = opendir(szPath)))) + { + vRedeError(pstSavm->m_lErrno = 127, strerror(errno)); + return RC_FAIL; + } + + while(pr = readdir(dir)) + { + if (NULL == strcasestr(pr->d_name, ".sdb")) + continue; + + if(DT_REG != pr->d_type) + continue; + + memset(szFile, 0, sizeof(szFile)); + snprintf(szFile, sizeof(szFile), "%s/%s", szPath, pr->d_name); + if(RC_SUCC != lMountTable(pstSavm, szFile)) + { + fprintf(stderr, "Warning:restore table failed, %s", + sGetTError(pstSavm->m_lErrno)); + continue; + } + } + + closedir(dir); + + return RC_SUCC; +} /**************************************************************************************** code end diff --git a/src/queue.c b/src/queue.c index a293937..5b93ef3 100644 --- a/src/queue.c +++ b/src/queue.c @@ -57,13 +57,18 @@ long _lDeleteQueue(SATvm *pstSavm, void *pvAddr) continue; pstSavm->m_lEffect ++; + SET_DATA_TRUCK(pstTruck, DATA_TRUCK_NULL); if(0 > (int)__sync_sub_and_fetch(&pv->m_lValid, 1)) { __sync_fetch_and_add(&pv->m_lValid, 1); break; } + } - SET_DATA_TRUCK(pstTruck, DATA_TRUCK_NULL); + if(0 == pstSavm->m_lEffect) + { + pstSavm->m_lErrno = NO_DATA_FOUND; + return RC_FAIL; } return RC_SUCC; @@ -193,7 +198,7 @@ retry: true --success false --time out *************************************************************************************************/ -bool bIsTimeOut(Timesp *tm, Timesp *tb) +bool bTimeOut(Timesp *tm, Timesp *tb) { Timesp tms; @@ -243,13 +248,13 @@ bool bIsTimeOut(Timesp *tm, Timesp *tb) RC_SUCC --success RC_FAIL --failure *************************************************************************************************/ -long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, Timesp *tm, size_t *plOut, +long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, time_t lTime, size_t *plOut, void **ppsvOut) { - Timesp tms; - int nPos, i = 0; - SHTruck *ps = NULL; extern int errno; + SHTruck *ps = NULL; + int nPos, i = 0; + Timesp tms, tm = {0}; TblDef *pv = (TblDef *)pvAddr; if(NULL == (*ppsvOut = (char *)malloc(lExpect * pv->m_lReSize))) @@ -257,10 +262,10 @@ long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, Timesp *tm, size_t pstSavm->m_lErrno = MALLC_MEM_ERR; return RC_FAIL; } - - for (*plOut = 0, clock_gettime(CLOCK_REALTIME, &tms); *plOut < lExpect; ) + + for (tm.tv_sec = lTime, *plOut = 0, clock_gettime(CLOCK_REALTIME, &tms); *plOut < lExpect; ) { - if(!bIsTimeOut(tm, &tms)) + if(lTime > 0 && !bTimeOut(&tm, &tms)) { pstSavm->m_lEffect = *plOut; if(0 == pstSavm->m_lEffect) @@ -273,7 +278,7 @@ long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, Timesp *tm, size_t return RC_FAIL; } - if(0 != Futex(&pv->m_lValid, FUTEX_WAIT, 0, tm)) + if(0 != Futex(&pv->m_lValid, FUTEX_WAIT, 0, &tm)) { if(ETIMEDOUT == errno) { @@ -296,16 +301,48 @@ long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, Timesp *tm, size_t ; } - if(0 == pv->m_lValid) - continue; + if(lTime == 0) + { + if(0 == pv->m_lValid) + { + pstSavm->m_lEffect = *plOut; + if(0 == pstSavm->m_lEffect) + { + TFree(*ppsvOut); + pstSavm->m_lErrno = NO_DATA_FOUND; + return RC_FAIL; + } - if(0 > (int)__sync_sub_and_fetch(&pv->m_lValid, 1)) - { - __sync_fetch_and_add(&pv->m_lValid, 1); - continue; + return RC_SUCC; + } + + if(0 > (int)__sync_sub_and_fetch(&pv->m_lValid, 1)) + { + __sync_fetch_and_add(&pv->m_lValid, 1); + + pstSavm->m_lEffect = *plOut; + if(0 == pstSavm->m_lEffect) + { + TFree(*ppsvOut); + pstSavm->m_lErrno = NO_DATA_FOUND; + return RC_FAIL; + } + + return RC_SUCC; + } } - -retrys: + else + { + if(0 == pv->m_lValid) + continue; + + if(0 > (int)__sync_sub_and_fetch(&pv->m_lValid, 1)) + { + __sync_fetch_and_add(&pv->m_lValid, 1); + continue; + } + } +retrys: /* at least cost one vaild */ if(pv->m_lMaxRow > (nPos = __sync_add_and_fetch(&pv->m_lListOfs, 1))) ; @@ -337,7 +374,7 @@ retrys: } /************************************************************************************************* - description:pop data from queue + description:pop data from queue use tvmpop parameters: pstSavm --stvm handle psvOut --out data @@ -393,7 +430,7 @@ long lPop(SATvm *pstSavm, void *pvOut, Uenum eWait) { long lRet; RunTime *pstRun = NULL; - static Timesp tm = {0, 1}; + static Timesp tm = {0, 0}; if(!pstSavm) { @@ -440,7 +477,6 @@ long lPop(SATvm *pstSavm, void *pvOut, Uenum eWait) long lPopup(SATvm *pstSavm, size_t lExpect, time_t lTime, size_t *plOut, void **ppsvOut) { long lRet; - Timesp tm = {0}; RunTime *pstRun = NULL; if(!pstSavm) @@ -465,8 +501,7 @@ long lPopup(SATvm *pstSavm, size_t lExpect, time_t lTime, size_t *plOut, void return _lPopupByRt(pstSavm, lExpect, lTime, plOut, ppsvOut); } - tm.tv_sec = lTime; - lRet = _lPops(pstSavm, pstRun->m_pvAddr, lExpect, &tm, plOut, ppsvOut); + lRet = _lPops(pstSavm, pstRun->m_pvAddr, lExpect, lTime, plOut, ppsvOut); vTblDisconnect(pstSavm, pstSavm->tblName); return lRet; } diff --git a/src/stvm.c b/src/stvm.c index d4fdd54..35c2026 100644 --- a/src/stvm.c +++ b/src/stvm.c @@ -38,6 +38,8 @@ extern char **environ; extern long lShutdownTvm(); extern void vSetNode(char *s); extern long lStartupTvm(TBoot *pstBoot); +extern long lBackupTables(SATvm *pstSavm); +extern long lRestoreTables(SATvm *pstSavm); extern long lMountTable(SATvm *pstSavm, char *pszFile); extern long lUnuseDump(SATvm *pstSavm, TABLE t); @@ -4019,7 +4021,8 @@ long lExecuteTvm(SATvm *pstSavm, char *pszSQL) **************************************************************************************************/ long lStartSystem(TBoot *pstBoot, char *pszMode) { - Benum eMode = 0; + long i, n; + Benum eMode = 0, eRestore = 0; SATvm *pstSavm = (SATvm *)pGetSATvm(); if(0 != access(getenv("TVMCFG"), R_OK)) @@ -4029,26 +4032,50 @@ long lStartSystem(TBoot *pstBoot, char *pszMode) return RC_FAIL; } - if(pszMode && !strcmp(pszMode, "o")) // offline + for(i = 0, n = NULL == pszMode ? 0 : strlen(pszMode); i < n; i ++) { - fprintf(stdout, "Warning:TVM will start offline\n"); - fflush(stdout); - eMode = 1; + switch(pszMode[i]) + { + case 'o': // offline + fprintf(stdout, "Warning:STVM will start offline ...\n"); + eMode = 1; + break; + case 'r': + fprintf(stdout, "STVM will do not restore tables ...\n"); + eRestore = 1; + break; + default: + break; + } } - + if(!bIsTvmBoot()) { - if(RC_SUCC != lStartupTvm(pstBoot)) - { - fprintf(stderr, "failed to boot TVM, %s\n", sGetTError(pstSavm->m_lErrno)); - return RC_FAIL; - } - } + if(RC_SUCC != lStartupTvm(pstBoot)) + { + fprintf(stderr, "failed to boot TVM, %s\n", sGetTError(pstSavm->m_lErrno)); + return RC_FAIL; + } - if(RC_SUCC != lBootLocal(pstSavm, pstBoot, eMode)) + if(RC_SUCC != lBootLocal(pstSavm, pstBoot, eMode)) + { + fprintf(stderr, "failed to boot LIS, %s\n", sGetTError(pstSavm->m_lErrno)); + return RC_SUCC; + } + + if(0 == eRestore && lRestoreTables(pstSavm)) + { + fprintf(stderr, "restore tables failed, %s\n", sGetTError(pstSavm->m_lErrno)); + return RC_FAIL; + } + } + else { - fprintf(stderr, "failed to boot LIS, %s\n", sGetTError(pstSavm->m_lErrno)); - return RC_SUCC; + if(RC_SUCC != lBootLocal(pstSavm, pstBoot, eMode)) + { + fprintf(stderr, "failed to boot LIS, %s\n", sGetTError(pstSavm->m_lErrno)); + return RC_SUCC; + } } fprintf(stderr, "start TVM : (%s)\n", sGetTError(pstSavm->m_lErrno)); @@ -4105,6 +4132,12 @@ long lStopSystem(TBoot *pstBoot, char *pszApp) if(!bIsTvmBoot()) return RC_SUCC; + if(RC_SUCC != lBackupTables(pstSavm)) + { + fprintf(stderr, "backup tables failed, %s\n", sGetTError(pstSavm->m_lErrno)); + return RC_FAIL; + } + if(RC_SUCC != lShutdownTvm()) { fprintf(stderr, "showdown node failed, %s\n", sGetTError(pstSavm->m_lErrno)); diff --git a/src/tree.c b/src/tree.c index f2e6446..c0fa570 100644 --- a/src/tree.c +++ b/src/tree.c @@ -11084,10 +11084,9 @@ void _vDumpQueue(SATvm *pstSavm, void *pvAddr, FILE *fp) RC_SUCC --success RC_FAIL --failure *************************************************************************************************/ -long lDumpTable(SATvm *pstSavm, TABLE t) +long _lDumpTable(SATvm *pstSavm, TABLE t, char *pszFile) { FILE *fp = NULL; - char szFile[512]; RunTime *pstRun = NULL; if(!pstSavm) @@ -11096,7 +11095,6 @@ long lDumpTable(SATvm *pstSavm, TABLE t) return RC_FAIL; } - memset(szFile, 0, sizeof(szFile)); if(RC_SUCC != lInitSATvm(pstSavm, t)) return RC_FAIL; @@ -11109,13 +11107,13 @@ long lDumpTable(SATvm *pstSavm, TABLE t) return RC_FAIL; } - snprintf(szFile, sizeof(szFile), "%s/%d.sdb", getenv("TVMDBD"), t); - if(NULL == (fp = fopen(szFile, "wb"))) + if(NULL == (fp = fopen(pszFile, "wb"))) { pstSavm->m_lErrno = FILE_NOT_RSET; return RC_FAIL; } fwrite(pGetTblDef(t), sizeof(TblDef), 1, fp); + fwrite(&pstRun->m_lType, sizeof(pstRun->m_lType), 1, fp); pstSavm->lSize = lGetRowSize(t); if(TYPE_MQUEUE == pstRun->m_lType) @@ -11130,6 +11128,24 @@ long lDumpTable(SATvm *pstSavm, TABLE t) return RC_SUCC; } +/************************************************************************************************* + description:Dump table + parameters: + pstSavm --stvm handle + t --table + return: + RC_SUCC --success + RC_FAIL --failure + *************************************************************************************************/ +long lDumpTable(SATvm *pstSavm, TABLE t) +{ + char szFile[512]; + + memset(szFile, 0, sizeof(szFile)); + snprintf(szFile, sizeof(szFile), "%s/%d.sdb", getenv("TVMDBD"), t); + return _lDumpTable(pstSavm, t, szFile); +} + /************************************************************************************************* description:lMountTable parameters: @@ -11142,12 +11158,13 @@ long lDumpTable(SATvm *pstSavm, TABLE t) long lMountTable(SATvm *pstSavm, char *pszFile) { TblDef sf; + uint lType; FILE *fp = NULL; + ulong uTimes = 0; long lEffect = 0; void *pvData = NULL; RunTime *pstRun = NULL; RWLock *prwLock = NULL; - ulong uTimes = 0, lRet; if(!pszFile || !pstSavm || !strlen(pszFile)) { @@ -11162,12 +11179,31 @@ long lMountTable(SATvm *pstSavm, char *pszFile) } fread(&sf, sizeof(TblDef), 1, fp); + fread(&lType, sizeof(lType), 1, fp); if(RC_SUCC != lInitSATvm(pstSavm, sf.m_table)) - goto MOUNT_ERROR; + { + if(pstSavm->m_lErrno != TBL_NOT_FOUND) + goto MOUNT_ERROR; + sf.m_lGroup = 0; + sf.m_lValid = 0; + sf.m_lTreePos = 0; + sf.m_lTreeRoot = 0; + sf.m_lGroupPos = 0; + sf.m_lGroupRoot= 0; + sf.m_lListPos = 0; + sf.m_lListOfs = 0; + memcpy((void *)pGetTblDef(sf.m_table), (void *)&sf, sizeof(TblDef)); + if(RC_SUCC != _lCustomTable(pstSavm, sf.m_table, sf.m_lMaxRow, false, lType)) + goto MOUNT_ERROR; + } + if(NULL == (pstRun = (RunTime *)pInitHitTest(pstSavm, sf.m_table))) goto MOUNT_ERROR; + pstRun->m_lType = lType; + pstRun->m_lRowSize = sf.m_lReSize; + pstRun->m_lLocal = RES_LOCAL_SID; if(sf.m_lReSize != lGetRowSize(sf.m_table)) { vTblDisconnect(pstSavm, pstSavm->tblName); @@ -11183,6 +11219,7 @@ long lMountTable(SATvm *pstSavm, char *pszFile) } pstSavm->pstVoid = pvData; + pstSavm->tblName = sf.m_table; pstSavm->lSize = sf.m_lReSize; prwLock = (RWLock *)pGetRWLock(pstRun->m_pvAddr); if(RC_SUCC != pthread_rwlock_wrlock(prwLock))