zcy
/
firmware
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch '1.2.4'

pull/1/head
deffpuzzl 2018-06-23 23:53:39 +08:00
parent 7dffc83838 e2aa4f8af2
commit 69978dfdba
9 changed files with 280 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
README.MD
View File

@ -18,13 +18,13 @@ STVM（truck of Virtual memory table）是一个开源的使用ANSI C语言编
***使用须知***
* 表空间大小在创建时已经确定, 一旦创建不能随意修改大小。
* 多机集群无法使用事务。
* 数据无法持久化(宕机情况)。
* 数据无法持久化(宕机情况), 可以定时利用接口备份。
* 索引长度限制（默认64字节），修改需编译项目。
* 启动后不能修改表字段，但可以修改字段别名
建议使用场景：***数据库缓存、数据缓存、事件驱动***
不定时更新 https://my.oschina.net/deffpuzzl/blog/1627626
不定时更新 https://my.oschina.net/deffpuzzl/blog/1627626
## 1、编译
**编译项目**
@ -46,17 +46,20 @@ cd src目录 **./make**
* **TVMCFG=$(TVMDBD)/.tvm.run** 启动参数
使用方法:
如果不使用多机集群，无需配置域文件，缺省本地启动。
如果不使用多机集群，无需配置域文件，缺省本地启动(该版本对集群支持较弱， 时间有限，稍后更新，请悉知)。
启动系统
启动系统
> stvm -w
停止系统
> stvm -s
停止系统
> stvm -s
STVM也提供一个类型sqlpuls类型简单工具。
进入SQL界面, 该工具主要用来运维调试使用
> stvm SQL
进入SQL界面, 该工具主要用来运维调试使用，其中showmode显示模式ROW|COLUMN，showsize单次显示记录数，也可用封装该指令的脚本**msql**进入
> stvm SQL --showmode=row --showsize=8
批量执行M-SQL脚本**exsql**，也可直接执行
>stvm SQL --msql=msql.bat
域的维护
> stvm DOM
@ -67,7 +70,7 @@ STVM也提供一个类型sqlpuls类型简单工具。
配置域完成后，可以配置编译到$TVMCFG下
> stvm -c stvm.conf
然后在stvm -w启动即可
然后在 **stvm -w** 启动即可
建议以单机模式64位下运行，将性能发挥在极致。
@ -116,6 +119,12 @@ STVM也提供一个类型sqlpuls类型简单工具。
>* 1、新增queue网络同步异步接口
>* 1、新增将表中长时间无用数据导出备份接口
队列的创建与表类似:
队列不支持创建索引，队列创建有2种类型，基于表队列和字符串队列，详见API。
创建队列语法示例:
在M-SQL中执行 create que_user_info.def
**下本版本：**
* 0、新增分布式锁
* 1、新增多机资源共享方式。
@ -123,13 +132,3 @@ STVM也提供一个类型sqlpuls类型简单工具。
**不定时更新**
---
#### *常见错误集合* ####
```
Q、failed to boot TVM, Invalid parameter or sem has disappeared
A：echo "500 X X X" > /proc/sys/kernel/sem
```

2
demo/popup.c
View File

@ -29,7 +29,7 @@ int main(int argc, char *argv[])
lTime = time(NULL);
queuenull(pstSavm, sizeof(dbUser), QUEUE_USER_INFO);
if(RC_SUCC != lPopup(pstSavm, 3, 5, &lRows, (void **)&pstUser))
if(RC_SUCC != lPopup(pstSavm, 3, 0, &lRows, (void **)&pstUser))
{
fprintf(stderr, "Pop error:(%u)(%s), time:%ld, effect:%d\n", pstSavm->m_lErrno,
sGetTError(pstSavm->m_lErrno), time(NULL) - lTime, pstSavm->m_lEffect);

1
include/tmain.h
View File

@ -23,7 +23,6 @@
#define _HHH_TVM_DOMAIN_HHH__
#include "tvm.h"
typedef struct epoll_event epollevt;
typedef long (*FUNCEXEC)(SATvm *pstSavm, void *arg);

2
include/tstr.h
View File

@ -62,6 +62,8 @@
#include <dlfcn.h>
#include <stdbool.h>
#include <libgen.h>
#define _GNU_SOURCE
#include <dirent.h>
typedef short int sint;
typedef unsigned int uint;

19
include/tvm.h
View File

@ -328,9 +328,9 @@ typedef long CREATE;
#define IDX_FIELD(t, f, a) if(RC_SUCC != lAddIdxField(tbl, type, FPOS(t, f), FLEN(t, f), a, #f)) \
return RC_FAIL;
#define FIELD(t, f, a) if(RC_SUCC != lSetTableIdx(tbl, FPOS(t, f), FLEN(t, f), #f, a, CHK_SELECT)) \
return RC_FAIL;
return RC_FAIL;
#define FIELU(t, f, a) if(RC_SUCC != lSetTableIdx(tbl, FPOS(t, f), FLEN(t, f), #f, a, IDX_SELECT)) \
return RC_FAIL;
return RC_FAIL;
#define FIELR(t, f, a) if(RC_SUCC != lSetTableIdx(tbl, FPOS(t, f), FLEN(t, f), #f, a, RCD_SELECT)) \
return RC_FAIL;
#define FINISH return RC_SUCC;
@ -339,6 +339,15 @@ typedef long CREATE;
/*************************************************************************************************
Field assignment
*************************************************************************************************/
#define conditvoid(p,v,l,t) do{ \
p->stCond.uFldcmp = 0; \
p->stUpdt.uFldcmp = 0; \
p->lFind = 0; \
p->tblName = t; \
p->lSize = l; \
p->pstVoid = (void *)&(v); \
}while(0);
#define conditbind(p,v,t) do{ \
p->stCond.uFldcmp = 0; \
p->stUpdt.uFldcmp = 0; \
@ -367,6 +376,12 @@ typedef long CREATE;
p->pstVoid = NULL; \
}while(0);
#define queuevoid(p,v,l,t) do{ \
p->lSize = l; \
p->tblName = t; \
p->pstVoid = (void *)&v; \
}while(0);
#define queueinit(p,v,t) do{ \
p->lSize = sizeof(v); \
p->tblName = t; \

92
src/func.c
View File

@ -19,12 +19,14 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#define _GNU_SOURCE
#include "tvm.h"
/************************************************************************************************
function
************************************************************************************************/
extern void vCondInsInit(FdCond *pstCond, TABLE t);
extern long _lDumpTable(SATvm *pstSavm, TABLE t, char *pszFile);
/*************************************************************************************************
description：dump the unused
@ -100,6 +102,96 @@ long lUnuseDump(SATvm *pstSavm, TABLE t)
return RC_SUCC;
}
/*************************************************************************************************
description：exit and backup tables
parameters:
pstSavm --stvm handle
return:
RC_SUCC --success
RC_FAIL --failure
*************************************************************************************************/
long lBackupTables(SATvm *pstSavm)
{
size_t lOut = 0, i;
char szFile[512];
TIndex *pstIndex = NULL;
memset(szFile, 0, sizeof(szFile));
if(RC_SUCC != lExportTable(SYS_TVM_INDEX, &lOut, (void *)&pstIndex))
return RC_FAIL;
snprintf(szFile, sizeof(szFile), "%s/backup", getenv("TVMDBD"));
if(0 != mkdir(szFile, S_IRWXU | S_IRGRP))
{
if(EEXIST != errno)
{
vRedeError(pstSavm->m_lErrno = 127, strerror(errno));
return RC_FAIL;
}
}
for(i = 0; i < lOut; i ++)
{
if((TYPE_SYSTEM == pstIndex[i].m_lType || TYPE_INCORE == pstIndex[i].m_lType) &&
SYS_TVM_SEQUE != pstIndex[i].m_table)
continue;
memset(szFile, 0, sizeof(szFile));
snprintf(szFile, sizeof(szFile), "%s/backup/%d.sdb", getenv("TVMDBD"),
pstIndex[i].m_table);
_lDumpTable(pstSavm, pstIndex[i].m_table, szFile);
}
TFree(pstIndex);
return RC_SUCC;
}
/*************************************************************************************************
description：boot and restore tables
parameters:
pstSavm --stvm handle
return:
RC_SUCC --success
RC_FAIL --failure
*************************************************************************************************/
long lRestoreTables(SATvm *pstSavm)
{
DIR *dir;
struct dirent *pr;
char szPath[512], szFile[128];
memset(szPath, 0, sizeof(szPath));
memset(szFile, 0, sizeof(szFile));
snprintf(szPath, sizeof(szPath), "%s/backup", getenv("TVMDBD"));
if ((NULL == (dir = opendir(szPath))))
{
vRedeError(pstSavm->m_lErrno = 127, strerror(errno));
return RC_FAIL;
}
while(pr = readdir(dir))
{
if (NULL == strcasestr(pr->d_name, ".sdb"))
continue;
if(DT_REG != pr->d_type)
continue;
memset(szFile, 0, sizeof(szFile));
snprintf(szFile, sizeof(szFile), "%s/%s", szPath, pr->d_name);
if(RC_SUCC != lMountTable(pstSavm, szFile))
{
fprintf(stderr, "Warning:restore table failed, %s",
sGetTError(pstSavm->m_lErrno));
continue;
}
}
closedir(dir);
return RC_SUCC;
}
/****************************************************************************************
code end

81
src/queue.c
View File

@ -57,13 +57,18 @@ long _lDeleteQueue(SATvm *pstSavm, void *pvAddr)
continue;
pstSavm->m_lEffect ++;
SET_DATA_TRUCK(pstTruck, DATA_TRUCK_NULL);
if(0 > (int)__sync_sub_and_fetch(&pv->m_lValid, 1))
{
__sync_fetch_and_add(&pv->m_lValid, 1);
break;
}
}
SET_DATA_TRUCK(pstTruck, DATA_TRUCK_NULL);
if(0 == pstSavm->m_lEffect)
{
pstSavm->m_lErrno = NO_DATA_FOUND;
return RC_FAIL;
}
return RC_SUCC;
@ -193,7 +198,7 @@ retry:
true --success
false --time out
*************************************************************************************************/
bool bIsTimeOut(Timesp *tm, Timesp *tb)
bool bTimeOut(Timesp *tm, Timesp *tb)
{
Timesp tms;
@ -243,13 +248,13 @@ bool bIsTimeOut(Timesp *tm, Timesp *tb)
RC_SUCC --success
RC_FAIL --failure
*************************************************************************************************/
long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, Timesp *tm, size_t *plOut,
long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, time_t lTime, size_t *plOut,
void **ppsvOut)
{
Timesp tms;
int nPos, i = 0;
SHTruck *ps = NULL;
extern int errno;
SHTruck *ps = NULL;
int nPos, i = 0;
Timesp tms, tm = {0};
TblDef *pv = (TblDef *)pvAddr;
if(NULL == (*ppsvOut = (char *)malloc(lExpect * pv->m_lReSize)))
@ -257,10 +262,10 @@ long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, Timesp *tm, size_t
pstSavm->m_lErrno = MALLC_MEM_ERR;
return RC_FAIL;
}
for (*plOut = 0, clock_gettime(CLOCK_REALTIME, &tms); *plOut < lExpect; )
for (tm.tv_sec = lTime, *plOut = 0, clock_gettime(CLOCK_REALTIME, &tms); *plOut < lExpect; )
{
if(!bIsTimeOut(tm, &tms))
if(lTime > 0 && !bTimeOut(&tm, &tms))
{
pstSavm->m_lEffect = *plOut;
if(0 == pstSavm->m_lEffect)
@ -273,7 +278,7 @@ long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, Timesp *tm, size_t
return RC_FAIL;
}
if(0 != Futex(&pv->m_lValid, FUTEX_WAIT, 0, tm))
if(0 != Futex(&pv->m_lValid, FUTEX_WAIT, 0, &tm))
{
if(ETIMEDOUT == errno)
{
@ -296,16 +301,48 @@ long _lPops(SATvm *pstSavm, void *pvAddr, size_t lExpect, Timesp *tm, size_t
;
}
if(0 == pv->m_lValid)
continue;
if(lTime == 0)
{
if(0 == pv->m_lValid)
{
pstSavm->m_lEffect = *plOut;
if(0 == pstSavm->m_lEffect)
{
TFree(*ppsvOut);
pstSavm->m_lErrno = NO_DATA_FOUND;
return RC_FAIL;
}
if(0 > (int)__sync_sub_and_fetch(&pv->m_lValid, 1))
{
__sync_fetch_and_add(&pv->m_lValid, 1);
continue;
return RC_SUCC;
}
if(0 > (int)__sync_sub_and_fetch(&pv->m_lValid, 1))
{
__sync_fetch_and_add(&pv->m_lValid, 1);
pstSavm->m_lEffect = *plOut;
if(0 == pstSavm->m_lEffect)
{
TFree(*ppsvOut);
pstSavm->m_lErrno = NO_DATA_FOUND;
return RC_FAIL;
}
return RC_SUCC;
}
}
retrys:
else
{
if(0 == pv->m_lValid)
continue;
if(0 > (int)__sync_sub_and_fetch(&pv->m_lValid, 1))
{
__sync_fetch_and_add(&pv->m_lValid, 1);
continue;
}
}
retrys:
/* at least cost one vaild */
if(pv->m_lMaxRow > (nPos = __sync_add_and_fetch(&pv->m_lListOfs, 1)))
;
@ -337,7 +374,7 @@ retrys:
}
/*************************************************************************************************
description：pop data from queue
description：pop data from queue use tvmpop
parameters:
pstSavm --stvm handle
psvOut --out data
@ -393,7 +430,7 @@ long lPop(SATvm *pstSavm, void *pvOut, Uenum eWait)
{
long lRet;
RunTime *pstRun = NULL;
static Timesp tm = {0, 1};
static Timesp tm = {0, 0};
if(!pstSavm)
{
@ -440,7 +477,6 @@ long lPop(SATvm *pstSavm, void *pvOut, Uenum eWait)
long lPopup(SATvm *pstSavm, size_t lExpect, time_t lTime, size_t *plOut, void **ppsvOut)
{
long lRet;
Timesp tm = {0};
RunTime *pstRun = NULL;
if(!pstSavm)
@ -465,8 +501,7 @@ long lPopup(SATvm *pstSavm, size_t lExpect, time_t lTime, size_t *plOut, void
return _lPopupByRt(pstSavm, lExpect, lTime, plOut, ppsvOut);
}
tm.tv_sec = lTime;
lRet = _lPops(pstSavm, pstRun->m_pvAddr, lExpect, &tm, plOut, ppsvOut);
lRet = _lPops(pstSavm, pstRun->m_pvAddr, lExpect, lTime, plOut, ppsvOut);
vTblDisconnect(pstSavm, pstSavm->tblName);
return lRet;
}

63
src/stvm.c
View File

@ -38,6 +38,8 @@ extern char **environ;
extern long lShutdownTvm();
extern void vSetNode(char *s);
extern long lStartupTvm(TBoot *pstBoot);
extern long lBackupTables(SATvm *pstSavm);
extern long lRestoreTables(SATvm *pstSavm);
extern long lMountTable(SATvm *pstSavm, char *pszFile);
extern long lUnuseDump(SATvm *pstSavm, TABLE t);
@ -4019,7 +4021,8 @@ long lExecuteTvm(SATvm *pstSavm, char *pszSQL)
**************************************************************************************************/
long lStartSystem(TBoot *pstBoot, char *pszMode)
{
Benum eMode = 0;
long i, n;
Benum eMode = 0, eRestore = 0;
SATvm *pstSavm = (SATvm *)pGetSATvm();
if(0 != access(getenv("TVMCFG"), R_OK))
@ -4029,26 +4032,50 @@ long lStartSystem(TBoot *pstBoot, char *pszMode)
return RC_FAIL;
}
if(pszMode && !strcmp(pszMode, "o")) // offline
for(i = 0, n = NULL == pszMode ? 0 : strlen(pszMode); i < n; i ++)
{
fprintf(stdout, "Warning:TVM will start offline\n");
fflush(stdout);
eMode = 1;
switch(pszMode[i])
{
case 'o': // offline
fprintf(stdout, "Warning:STVM will start offline ...\n");
eMode = 1;
break;
case 'r':
fprintf(stdout, "STVM will do not restore tables ...\n");
eRestore = 1;
break;
default:
break;
}
}
if(!bIsTvmBoot())
{
if(RC_SUCC != lStartupTvm(pstBoot))
{
fprintf(stderr, "failed to boot TVM, %s\n", sGetTError(pstSavm->m_lErrno));
return RC_FAIL;
}
}
if(RC_SUCC != lStartupTvm(pstBoot))
{
fprintf(stderr, "failed to boot TVM, %s\n", sGetTError(pstSavm->m_lErrno));
return RC_FAIL;
}
if(RC_SUCC != lBootLocal(pstSavm, pstBoot, eMode))
if(RC_SUCC != lBootLocal(pstSavm, pstBoot, eMode))
{
fprintf(stderr, "failed to boot LIS, %s\n", sGetTError(pstSavm->m_lErrno));
return RC_SUCC;
}
if(0 == eRestore && lRestoreTables(pstSavm))
{
fprintf(stderr, "restore tables failed, %s\n", sGetTError(pstSavm->m_lErrno));
return RC_FAIL;
}
}
else
{
fprintf(stderr, "failed to boot LIS, %s\n", sGetTError(pstSavm->m_lErrno));
return RC_SUCC;
if(RC_SUCC != lBootLocal(pstSavm, pstBoot, eMode))
{
fprintf(stderr, "failed to boot LIS, %s\n", sGetTError(pstSavm->m_lErrno));
return RC_SUCC;
}
}
fprintf(stderr, "start TVM : (%s)\n", sGetTError(pstSavm->m_lErrno));
@ -4105,6 +4132,12 @@ long lStopSystem(TBoot *pstBoot, char *pszApp)
if(!bIsTvmBoot()) return RC_SUCC;
if(RC_SUCC != lBackupTables(pstSavm))
{
fprintf(stderr, "backup tables failed, %s\n", sGetTError(pstSavm->m_lErrno));
return RC_FAIL;
}
if(RC_SUCC != lShutdownTvm())
{
fprintf(stderr, "showdown node failed, %s\n", sGetTError(pstSavm->m_lErrno));

51
src/tree.c
View File

@ -11084,10 +11084,9 @@ void _vDumpQueue(SATvm *pstSavm, void *pvAddr, FILE *fp)
RC_SUCC --success
RC_FAIL --failure
*************************************************************************************************/
long lDumpTable(SATvm *pstSavm, TABLE t)
long _lDumpTable(SATvm *pstSavm, TABLE t, char *pszFile)
{
FILE *fp = NULL;
char szFile[512];
RunTime *pstRun = NULL;
if(!pstSavm)
@ -11096,7 +11095,6 @@ long lDumpTable(SATvm *pstSavm, TABLE t)
return RC_FAIL;
}
memset(szFile, 0, sizeof(szFile));
if(RC_SUCC != lInitSATvm(pstSavm, t))
return RC_FAIL;
@ -11109,13 +11107,13 @@ long lDumpTable(SATvm *pstSavm, TABLE t)
return RC_FAIL;
}
snprintf(szFile, sizeof(szFile), "%s/%d.sdb", getenv("TVMDBD"), t);
if(NULL == (fp = fopen(szFile, "wb")))
if(NULL == (fp = fopen(pszFile, "wb")))
{
pstSavm->m_lErrno = FILE_NOT_RSET;
return RC_FAIL;
}
fwrite(pGetTblDef(t), sizeof(TblDef), 1, fp);
fwrite(&pstRun->m_lType, sizeof(pstRun->m_lType), 1, fp);
pstSavm->lSize = lGetRowSize(t);
if(TYPE_MQUEUE == pstRun->m_lType)
@ -11130,6 +11128,24 @@ long lDumpTable(SATvm *pstSavm, TABLE t)
return RC_SUCC;
}
/*************************************************************************************************
description：Dump table
parameters:
pstSavm --stvm handle
t --table
return:
RC_SUCC --success
RC_FAIL --failure
*************************************************************************************************/
long lDumpTable(SATvm *pstSavm, TABLE t)
{
char szFile[512];
memset(szFile, 0, sizeof(szFile));
snprintf(szFile, sizeof(szFile), "%s/%d.sdb", getenv("TVMDBD"), t);
return _lDumpTable(pstSavm, t, szFile);
}
/*************************************************************************************************
description：lMountTable
parameters:
@ -11142,12 +11158,13 @@ long lDumpTable(SATvm *pstSavm, TABLE t)
long lMountTable(SATvm *pstSavm, char *pszFile)
{
TblDef sf;
uint lType;
FILE *fp = NULL;
ulong uTimes = 0;
long lEffect = 0;
void *pvData = NULL;
RunTime *pstRun = NULL;
RWLock *prwLock = NULL;
ulong uTimes = 0, lRet;
if(!pszFile || !pstSavm || !strlen(pszFile))
{
@ -11162,12 +11179,31 @@ long lMountTable(SATvm *pstSavm, char *pszFile)
}
fread(&sf, sizeof(TblDef), 1, fp);
fread(&lType, sizeof(lType), 1, fp);
if(RC_SUCC != lInitSATvm(pstSavm, sf.m_table))
goto MOUNT_ERROR;
{
if(pstSavm->m_lErrno != TBL_NOT_FOUND)
goto MOUNT_ERROR;
sf.m_lGroup = 0;
sf.m_lValid = 0;
sf.m_lTreePos = 0;
sf.m_lTreeRoot = 0;
sf.m_lGroupPos = 0;
sf.m_lGroupRoot= 0;
sf.m_lListPos = 0;
sf.m_lListOfs = 0;
memcpy((void *)pGetTblDef(sf.m_table), (void *)&sf, sizeof(TblDef));
if(RC_SUCC != _lCustomTable(pstSavm, sf.m_table, sf.m_lMaxRow, false, lType))
goto MOUNT_ERROR;
}
if(NULL == (pstRun = (RunTime *)pInitHitTest(pstSavm, sf.m_table)))
goto MOUNT_ERROR;
pstRun->m_lType = lType;
pstRun->m_lRowSize = sf.m_lReSize;
pstRun->m_lLocal = RES_LOCAL_SID;
if(sf.m_lReSize != lGetRowSize(sf.m_table))
{
vTblDisconnect(pstSavm, pstSavm->tblName);
@ -11183,6 +11219,7 @@ long lMountTable(SATvm *pstSavm, char *pszFile)
}
pstSavm->pstVoid = pvData;
pstSavm->tblName = sf.m_table;
pstSavm->lSize = sf.m_lReSize;
prwLock = (RWLock *)pGetRWLock(pstRun->m_pvAddr);
if(RC_SUCC != pthread_rwlock_wrlock(prwLock))