1.0.5 • Published 18 days ago
nd-aloc v1.0.5
nd-aloc
- only work in nodejs
- some util for test nvlang'runtime allocator
install
- npm install nd-aloc
splitted
usage
const x = require("nd-aloc");example
calloc
default is 1G
var mem = x.calloc();
> mem.is_succ()
true
>
> mem.bytsz
1073741824
>
> mem.ptr.toString(16)
'7f6a80000000'
>
> mem.read(0,100)
ArrayBuffer {
[Uint8Contents]: <00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00>,
byteLength: 100
}
>
mem.write([1,2,3,4,5,6,7,8],8)
> mem.read(0,100)
ArrayBuffer {
[Uint8Contents]: <00 00 00 00 00 00 00 00 01 02 03 04 05 06 07 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00>,
byteLength: 100
}
>
> mem.free()
[ true, null ]
>
> mem.is_already_freed()
true
> mem.free()
[ false, 'shmem can NOT be free, can only be destroyed' ]
>
//cant read after freeread write load dump
var mem = x.calloc(2,x.PAGE_4K);
> mem.read(0,32)
ArrayBuffer {
[Uint8Contents]: <00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00>,
byteLength: 32
}
>
var f64a = new Float64Array(1);
for(let i=0;i<mem.bytsz;i+=8) {
f64a[0] = Math.random();
mem.write(f64a,i);
}
/*
> mem.read()
ArrayBuffer {
[Uint8Contents]: <60 00 c2 60 ee 5f da 3f b0 89 b6 88 da 5c d2 3f 98 d3 06 16 08 05 cb 3f 00 2d 42 5d b4 1d c1 3f 10 b9 b9 28 80 50 d1 3f 94 2c e2 3c 3d 9e e2 3f c2 d4 3e 56 1a 29 e1 3f 80 88 ea 2b 09 9b d2 3f 4e 28 e2 d4 86 15 e2 3f 70 e9 b4 51 7b 82 e3 3f ce 6a 52 2d c8 3e e7 3f 0c e2 88 21 c0 78 e4 3f aa cd 09 7f ... 8092 more bytes>,
byteLength: 8192
}
*/
// sync blocked
mem.dump_to_file("dump.f64a")
mem.free()
var mem = x.calloc_mem_fr_file("dump.f64a",x.PAGE_4K);
var ab = mem.read();
var f64a = new Float64Array(ab);
/*
> f64a
Float64Array(1024) [
0.41210517357831655, 0.2869173369491831, 0.21109105182190713,
0.1337190108646169, 0.27053836800668396, 0.5818163098459928,
.....
]
*/
> mem.reset()
> mem.read()
ArrayBuffer {
[Uint8Contents]: <00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... 8092 more bytes>,
byteLength: 8192
}
> f64a
Float64Array(1024) [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0,
... 924 more items
]
> kvstore (IF mem is NOT fixed-addr-shmem, cant be Dumped)
IF kv-pairs count < 5000000, ITS iterator is a little SLOW than using {} directlty;
BUT it can store VERY-MANY kv-pairs , just depends on memory
var mem = x.calloc(1024*256,x.PAGE_4K);
var kv = mem.to_kv_str_store();
> kv[x.kv_sym_size]()
0
> kv.a = 'its-a-string'
'its-a-string'
> kv[x.kv_sym_size]()
1
> kv.a
'its-a-string'
> kv[x.kv_sym_rplc_key]("a","A")
true
> kv.a
undefined
> kv.A
'its-a-string'
>
> kv[x.kv_sym_size]()
1
>
> kv[x.kv_sym_clear]()
undefined
> kv[x.kv_sym_size]()
0
> kv.A
kv.a = 'aa'; kv.b='bb'; kv.c='cc';
kv[0] = '00'; kv[1]='11'; kv[2]='22';
kv.a = 'aa'; kv.b='bb'; kv.c='cc';
kv[0] = '00'; kv[1]='11'; kv[2]='22';
kv[x.kv_sym_forof]((val,key,idx,self)=>{
console.log(val,key,idx);
return(true); // false means break
});
/*
00 0 0
11 1 1
22 2 2
aa a 3
bb b 4
cc c 5
*/
// PERFORMANCE
//// <small-key>
root@dev:/home/cu-lib/nd-aloc/TEST# cat insert.js
var x = require("../index")
var mem = x.calloc(1024*256,x.PAGE_4K);
(()=>{
var kv = mem.to_kv_str_store();
var insert = (cnt)=> {
var si = new Date;
for(let i=0;i<cnt;++i) {
var k = "k"+i;
var v = "v"+i;
kv[k] = v;
}
var ei = new Date;
console.log((ei-si)/1000)
}
var CNT = Number(process.argv[2]);
insert(CNT);
kv.k0;
console.log(kv.k0,kv[`k${CNT-1}`])
})();
//mem.free();
root@dev:/home/cu-lib/nd-aloc/TEST#
root@dev:/home/cu-lib/nd-aloc/TEST# node insert.js 20000000
37.974
v0 v19999999
root@dev:/home/cu-lib/nd-aloc/TEST# node
Welcome to Node.js v20.1.0.
Type ".help" for more information.
> 20000000 /37.974
526676.1468373098
>
// <big-key>
root@dev:/home/cu-lib/nd-aloc/TEST# cat insert.js
var x = require("../index")
var mem = x.calloc(1024*256,x.PAGE_4K);
(()=>{
var kv = mem.to_kv_str_store();
var insert = (cnt)=> {
var si = new Date;
for(let i=0;i<cnt;++i) {
var k = "abcdefghijklmnopqrstuvwxyz_"+i;
var v = "v"+i;
kv[k] = v;
}
var ei = new Date;
console.log((ei-si)/1000)
}
var CNT = Number(process.argv[2]);
insert(CNT);
kv.k0;
console.log(kv.abcdefghijklmnopqrstuvwxyz_0,kv[`abcdefghijklmnopqrstuvwxyz_${CNT-1}`])
})();
//mem.free();
root@dev:/home/cu-lib/nd-aloc/TEST# node insert.js 20000000
45.024
v0 v19999999
root@dev:/home/cu-lib/nd-aloc/TEST# node
Welcome to Node.js v20.1.0.
Type ".help" for more information.
> 20000000 /45.024
444207.53375977254
>
//shmem kvstore (same as kvstore, but can be Dump, can be used by other process(you need to impl sync-mechanism))
var shmem = x.creat_fixed_addr_shmem("./TEST/shmem-kv");
/*
> shmem.shmid
59
>
*/
var kv = shmem.to_kv_str_store();
kv.k0 = ""
kv.k1 = "a"
kv.k2 = "ab"
kv.k3 = "abc"
kv.k4 = "abcd"
kv.k5 = "abcde"
kv[x.kv_sym_forof]((v,k,i)=>{console.log(v,k,i); return(true)});
// EXIT OR from new process
> .exit
var shmem = x.load_fixed_addr_shmem_fr_shmid(59);
var kv = shmem.as_kv_str_store();
kv[x.kv_sym_forof]((v,k,i)=>{console.log(v,k,i); return(true)});
/*
k0 0
a k1 1
ab k2 2
abc k3 3
abcd k4 4
abcde k5 5
*/u32:u32 store this mode need at least 1207959600 bytes memory
its store edge-pair: [uint32 uint32]
its just similiar to :
{
111: new Set(222,333,444,555),
222: new Set(xxx,xxx....),
....
}
can store many, max 67108864 edges
its key AND value can NOT be 0, MUST between [1,16777216)
var u32u32 = x.calloc_u32u32_store();
/*
> u32u32.bucket_cnt_
16777216
> u32u32.max_edge_cnt_
67108864
> u32u32.lefted_edge_cnt_
67108864
> u32u32.used_edge_cnt_
0
> u32u32.idtype_
'uin32_t'
>
*/
> u32u32.add(111,222)
true
>
> u32u32.add(222,111)
true
>
> u32u32.has(111,222)
true
> u32u32.has(222,111)
true
> u32u32.del(222,111)
true
>
> u32u32.has(222,111)
false
>
> u32u32.add(111,333)
true
> u32u32.add(111,444)
true
> u32u32.add(111,555)
true
>
> u32u32.used_edge_cnt_
4
>
> u32u32.get_dsts_cnt(111)
4
> u32u32.get_all_dsts(111)
Uint32Array(4) [ 222, 333, 444, 555 ]
>
/*
> u32u32.for_each_dst(111,(src,dst)=>{console.log(dst)})
222
333
444
555
*/
> u32u32.$$[111]
Uint32Array(4) [ 222, 333, 444, 555 ]
> u32u32.$$[222]
Uint32Array(0) []
>
var perf_add = ()=>{
var si = new Date;
for(var i =1;i<16777215;++i) {
u32u32.add(i,16777215)
}
var ei = new Date;
console.log((ei-si)/1000)
}
perf_add();
> 0.494 // about 3000_0000 /second
> u32u32.used_edge_cnt_
16777214
>
var perf_srch = ()=>{
var si = new Date;
var cnt = 0;
u32u32.foreach((src,dst)=>{++cnt})
var ei = new Date;
console.log((ei-si)/1000)
}
perf_srch();
//1.681
var perf_del = ()=>{
var si = new Date;
for(var i =1;i<16777215;++i) {
u32u32.del_all_dsts(i)
}
var ei = new Date;
console.log((ei-si)/1000)
}
> perf_del()
0.573
>
> u32u32.used_edge_cnt_
0
> u32u32 using shmem
> var [cond,u32u32] = x.creat_shared_u32u32_store("./TEST/u32u32")
> u32u32
U32U32 {
mem: Mem(6) [
4096,
60, //================>
140055393861632,
1342177280,
'/home/cu-lib/nd-aloc/TEST/u32u32',
-1,
kvstore_type: 'u32u32'
]
}
> u32u32.add(1,5)
true
> u32u32.$$[1]
Uint32Array(1) [ 5 ]
>
>.exit
# ipcs -m
------ Shared Memory Segments --------
key shmid owner perms bytes nattch status
0x41028fab 51 root 0 3892314408 0
0x41029d9f 60 root 0 1342177280 0 //===========================================>
> var [cond,u32u32] = x.load_shared_u32u32_store(60)
> u32u32
U32U32 {
mem: Mem(6) [
0,
60,
140416640876544,
1342177280,
null,
-1,
kvstore_type: 'u32u32'
]
}
> u32u32.$$[1]
Uint32Array(1) [ 5 ]
> vloc (aloc any byte size)
var vloc = x.calloc_vloc_store();
var ab = vloc.aloc(32)
/*
ArrayBuffer {
[Uint8Contents]: <00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00>,
byteLength: 32
}
> vloc.show_brief()
{
SELF_BYTSZ : 1107296320 ,
MAX_DATA_CNT : 1048576 ,
MAX_DATA_BYTSZ : 1073741824 ,
node_cnt : 1 ,
node_cursor : 1 ,
fst_rng_node_idx : 1 ,
aloced_bytsz : 32
lefted_bytsz : 1073741792
}
> vloc.show_nds()
{si: 32,ei: 1073741824}
*/
vloc.free(ab)
/*
> ab
ArrayBuffer { (detached), byteLength: 0 }
>
> vloc.show_brief()
{
SELF_BYTSZ : 1107296320 ,
MAX_DATA_CNT : 1048576 ,
MAX_DATA_BYTSZ : 1073741824 ,
node_cnt : 1 ,
node_cursor : 1 ,
fst_rng_node_idx : 1 ,
aloced_bytsz : 0
lefted_bytsz : 1073741824
}
> vloc.show_nds()
{si: 0,ei: 1073741824}
>
*/METHODS
mem.reset();
mem._write mem.bytsz mem.constructor mem.destroy_if_is_shmem mem.dump_to_file mem.free
mem.is_already_freed mem.is_cerrno_err mem.is_fn_cant_be_opened_err mem.is_ftok_err mem.is_load_from_shmid_err mem.is_ptr_negative_err
mem.is_shmem mem.is_shmem_fn_already_exist_err mem.is_shmem_fn_creat_err mem.is_succ mem.page_sz mem.ptr
mem.read mem.reload_all_data_fr_file mem.shmfn mem.shmid mem.write
mem.to_kv_store(value_typnm="double" : "double"|"string"|"bool", reset=false) -> kvstore
// clear mem and creat new kvstore
mem.to_kv_f64_store() -> kvstore<std::pmr::string,double>
mem.to_kv_bl_store() -> kvstore<std::pmr::string,bool>
mem.to_kv_str_store() -> kvstore<std::pmr::string,string>
mem.as_kv_store(value_typnm="double" : "double"|"string"|"bool") -> kvstore
// keep old mem AND load as kvstore , only work on fixed-addr-shmem
mem.as_kv_f64_store()
mem.as_kv_bl_store()
mem.as_kv_str_store()
kvstore<std::pmr::string,VT>[<string>] @getter ->VT
kvstore<std::pmr::string,VT>[<string>] =<VT> @setter ->VT
delete kvstore<std::pmr::string,VT>[<string>] @deleter ->Boolean
kvstore[kv_sym_size]() // pair-count of the std::pmr::map
kvstore[kv_sym_rplc_key](rplc_key, old_key:String, new_key:String) // replace key
kvstore[kv_sym_clear]() // clear()
kvstore[kv_sym_forof](
(v,k,i,self)=>bool@[if true: continue else break]
)
kvstore[kv_sym_get_mem](); // get the mem APIS
HUGE_MODE_NONE: 0,
HUGE_MODE_2M: 1,
HUGE_MODE_1G: 2,
PAGE_4K: 4096,
PAGE_2M: 2097152,
PAGE_1G: 1073741824,
calloc: [Function: calloc],
calloc_mem_fr_file: [Function: calloc_mem_fr_file],
_creat_shmem,
creat_shmem: [Function: creat_shmem],
creat_shmem_fr_file: [Function: creat_shmem_fr_file],
creat_fixed_addr_shmem,
creat_fixed_addr_shmem_fr_file,
_load_shmem_fr_shmid,
load_shmem_fr_shmid: [Function: load_shmem_fr_shmid]
load_fixed_addr_shmem_fr_shmid,
kv_sym_get_mem: Symbol(kv_get_mem),
kv_sym_size: Symbol(kv_size),
kv_sym_rplc_key: Symbol(kv_rplc_key),
kv_sym_clear: Symbol(kv_clear),
kv_sym_forof: Symbol(kv_forof),
creat_shared_str_kv_store: [Function: creat_shared_str_kv_store],
creat_shared_bl_kv_store: [Function: creat_shared_bl_kv_store],
creat_shared_f64_kv_store: [Function: creat_shared_f64_kv_store],
load_shared_str_kv_store: [Function: load_shared_str_kv_store],
load_shared_bl_kv_store: [Function: load_shared_bl_kv_store],
load_shared_f64_kv_store: [Function: load_shared_f64_kv_store]LICENSE
- ISC