nd-fast-walk-dir v1.2.21

nd-fast-walk-dir

• fast recusive walk dir: about 360000 paths per/second, recursive walk the whole "/" cost 2.5 - 3.5 second (based on your CPU/DISK)
• 5 times faster than fs.readdirSync
• only work for linux

• need g++-14 installed

install

• npm install nd-fast-walk-dir

RESTRICT

• this script SOMETIMES can NOT correctly walk /run/systemd/incoming AND /var/tmp

usage

  const walk   = require("nd-fast-walk-dir");

example

simple walk only for path

  these methods:
        will ONLY just search one hardlink of a inode (IF multiple hardlinks other than .|.. exists)
        are  slightly faster than non-simple-methods 
        cost much less memory than non-simple-methods

 using ftype_t = "dire" | "file" | "link" | "sock" | "fifo" | "char" | "blok"; 

 xxxx : "dire" | "file" | "link" | "sock" | "fifo" | "char" | "blok"

 list_all_[@xxxx] : (
    dir_name : String,
    filter   : (type: ftype_t,  full_path:String) -> Boolean
 )->Array<String>

  var files = walk.list_all_file("/",(full_path)=>true)
  var dires = walk.list_all_dire("/",(full_path)=>true)
  var links = walk.list_all_link("/",(full_path)=>true)               // soft-link  NOT hard-link
  var socks = walk.list_all_sock("/",(full_path)=>true)
  var fifos = walk.list_all_fifo("/",(full_path)=>true)
  var chars = walk.list_all_char("/",(full_path)=>true)
  var bloks = walk.list_all_blok("/",(full_path)=>true)


  .list_all_paths_of_type(
     dir_name : String,
     type     :  ftype_t
     filter   : (full_path:String) -> Boolean
  ) -> Array<String>

  .path_only_walk(
     dir_name : String,
     filter   : (type: ftype_t,  full_path:String) -> Boolean,
  ) -> Array<Tuple[2]<ftype_t,String>>

  .gen_path_only_walk(
      dir_name : String,
      filter   : (type: ftype_t,  full_path:String) -> Boolean,
      enableHugeCount : false ,                                   // if your files exceed 2**32 , set this to true:       false: will using Uint32Array , true: will using BigUint64Array
  )->SyncGenerator<Tuple[2]<ftype_t,String>>;

  

SLOW verbosed GRAPH BUILDER : detailed_sync_snapshot_graph

  this will make a GRAPH from filesystem:
     it will keep              all hard-links
             keep              soft-link-to
             build-and-keep    soft-links-from
     IF the soft-link-to IS external of the-walking-dir :  
        for example: 
            your walk  "/home/data"
            /home/data/softlink   pointer to  /root/something
       it will just keep a  string "/root/something"       
    ELSE  it will keep a GraphUnit

 the GRAPH is READONLY  



 examples:
    
        var rt = walk.detailed_sync_snapshot_graph("/");

        > rt.fs
        FileSystemGraphUnitSnapShot [/ :: 

            run
            var
            mnt
            lost+found
            lib.usr-is-merged
            sbin.usr-is-merged
            .....
            bin -> usr/bin
            swap.img
            etc

        ] {
          cnt: 1149087,
          graph_unit_vec_offset: 48,
          chid_vec_offset: 55156224,
          name_buffer_offset: 59752568,
          dir_name: '/'
        }


        > rt
         [/ :: 

            run
            var
            mnt
            lost+found
            lib.usr-is-merged
            sbin.usr-is-merged
            ....
            lib64 -> usr/lib64
            bin -> usr/bin
            swap.img
            etc

         ] {}
        > 

        > rt.type
        'dire'
        > rt.hlinks.map(r=>r.path)
        [
          '/',
          '/run/initramfs',
          '/dev/rfkill',
          '/sys/fs',
          '/run/lock/lvm',
          .....
          '/snap/snapd/23258/etc/apparmor.d',
          '/proc/sys/fs/binfmt_misc/status'
        ]
        > rt.lnfrs.map(r=>r.path)
        [
          '/proc/1/cwd',  '/proc/1/root',  '/proc/2/cwd',  '/proc/2/root',
          '/proc/3/cwd',  '/proc/3/root',  '/proc/4/cwd',  '/proc/4/root',
          .....
          '/proc/58/cwd', '/proc/58/root', '/proc/59/cwd', '/proc/59/root',
          '/proc/60/cwd', '/proc/60/root', '/proc/61/cwd', '/proc/61/root',
          '/proc/62/cwd', '/proc/62/root', '/proc/63/cwd', '/proc/63/root',
          ... 1021 more items
        ]
        >  
   

         .detailed\_sync\_snapshot\_graph(
             dir\_name : String,
             BuildRelationAfterCtor =true               // IF set this to False , it will be much faster, BUT can NOT resolve soft-link-to/soft-link-from/all-hard-links
         )-\><GraphUnit>

         GraphUnit:
            
            .fs             @getter  -> <FileSystemGraphUnitSnapShot>
            .type           @getter  -> ftype_t 
            .is_leaf        @getter  ->  Boolean   // NOT-DIRECTORY OR Empty-Directory 
            .id             @getter  ->  Integer
            .chcnt          @getter  ->  Integer  
            .depth          @getter  ->  Integer                                   
            .hlink_cnt      @getter  ->  Integer   // st_nlink  hard-link-count(NOT include . AND ..)
            .ino            @getter  ->  Integer   // real-fs-inode 
            .lno            @getter  ->  Integer   // real-fs-soft-link-to-inode  
            .slink_from_cnt @getter  ->  Integer     
            .name           @getter  ->  String
            .path           @getter  ->  String   



            .pr             @getter  ->  <GraphUnit> |Null   //parent     null means relative root                   
            .children       @getter  ->  Array<GraphUnit>       
            .hlinks         @getter  ->  Array<GraphUnit|Undefined>                               //Undefined means some error happened     
            .slink          @getter  ->  <GraphUnit> |  String@LinkToExternal | Undefined         // soft-link-to 
            .slinks_from    @getter  ->  Array<GraphUnit>                                         // soft-link-from NOT include external-link-from        



            .get_brief_str()        -> String                   //readable string
            .child(AryIndex|String) -> <GraphUnit> | Null
            .json()                 -> JSON                     // this is slow: recursive build to json                    

       FileSystemGraphUnitSnapShot: its a array buffer:
                cnt;
                graph_unit_vec_offset;
                chid_vec_offset;
                name_buffer_offset;
                dir_name

fast walk return a array

        var all = walk(
           "/",                                           //directory to walk 
          (type,full\_path,rel\_depth)=\>true,            //filter function 
          {
             max\_rel\_depth: 2048 ,         // MAX relative depth to directory to walk, default is 2048
             IgnoreProcDevRunTmpSys:true,              // ignore  /proc AND /dev  AND /run AND /tmp AND /var/tmp
             IgnoreSockFifoCharBlok:true      // ignore  SOCK | FIFO | CHAR-DEV | BLOCK-DEV
          }
        )
        console.log(all.length)
        console.log(all[1000000])
        /*
            [
              'file',             // type <- "file" | "dire" | "link" | "sock" | "fifo" | "char" | "blok" 
              '/home/cu-lib/GCC13GIT/gcc-13.2.0/gcc/testsuite/gcc.target/arm/mve/intrinsics/viwdupq\_m\_n\_u8.c',     //【full path】
              10
            ]
        */

slow detailed graph walk

   //---this will include  parent  AND link-to  AND inode infomation
   //---it will ignore  /proc AND /dev  AND /run AND /tmp AND /var/tmp
   //---it will skip   all sock | fifo | char-dev | blk-dev,      only keep file AND directory AND soft-link
   //---it will return a group of  functions
   //---it is slow:  need about 10 seconds    to  walk the whole "/" AND build graph snapshot

    walk.sync_snapshot_graph(dir_name="./",max_depth=2048) -> {
      list_all_paths: [Function: list_all_paths],
      list_all_inodes: [Function: list_all_inodes],
      list_all_types: [Function: list_all_types],
      list_all_rel_depths: [Function: list_all_rel_depths],
      list_show: [Function: list_show],
      ind: [Function: ind],
      fpath: [Function: fpath],
      pn: [Function: pn],
      type: [Function: type],
      parent: [Function: parent],
      children: [Function: children],
      get_linkto: [Function: children],
      foreach: [Function: foreach],
      filter: [Function: filter]        
    }

METHODS

    walk.sync_snapshot_graph(dir_name="./",max_depth=2048) -> {
      list_all_paths: [Function: list_all_paths],
      list_all_inodes: [Function: list_all_inodes],
      list_all_types: [Function: list_all_types],
      list_all_rel_depths: [Function: list_all_rel_depths],
      list_show: [Function: list_show],
      ind: [Function: ind],
      fpath: [Function: fpath],
      pn: [Function: pn],
      type: [Function: type],
      parent: [Function: parent],
      children: [Function: children],
      get_linkto: [Function: children],
      foreach: [Function: foreach],
      filter: [Function: filter]
    }

CONSTS

      type name:
          fifo
          char
          dire
          blok
          file
          link
          sock               

APIS

    (
        dir_name="./",
        fltr_func=(type,full_path,rel_depth)=>true,
        cfg = {
           max_rel_depth:2048,              // dir_name IS 0 
           IgnoreProcDevRunTmpSys:true,              // ignore  /proc AND /dev  AND /run AND /tmp AND /var/tmp
           IgnoreSockFifoCharBlok:true      // ignore  SOCK | FIFO | CHAR-DEV | BLOCK-DEV
        }
    ) -> Array< [type:FixedString, path:String, relative_depth:UInt]>

LICENSE

• ISC