Meteor job prcessing files freezes

#1

Hi all.

I am reading files with the bitcoin blockchain in order to proccess them.
Every time I restart meteor the process hangs in a different blockNumber (around block 350) all in the 1st blocks file.

This is the minimum code to reproduce the error:

import fs from 'fs'
import path from 'path'

function BlockDataCopier() { 
    let offset = 0
    let inFn = 0
    let inF = null // file descriptor
    let blkCountIn = 0

    function inFileName(fn) { // fn = file number.
        const fileName = `blk${(""+fn).padStart(5, '0')}.dat` // blk00000.dat, blk00001.dat, ...
        return path.join('/home/r2d2/vue/linear_blockchain', fileName)
    }

    while(true) { 
        if(!inF) {
            const fname = inFileName(inFn)
            console.log("Input file ", fname)
            try {
                inF = fs.openSync(fname, 'r')
                offset = 0
            } catch(e) {
                console.log("End of data", fname, e)
                return
            }
        }

        const inhdr = readSlice(8)
        if (!inhdr) { 
            fs.closeSync(inF)
            inF = null
            inFn += 1
            // if(inFn == 3) break // it works until file 3 when this line is uncommented ???
            continue 
        }
        const inLenLE = inhdr.readUInt32LE(4)
        inLen = inLenLE - 80 // length without header
        const blk_hdr = readSlice(80)

        const rawblock = readSlice(inLen)
        console.log("Processing blk", blkCountIn, "file:", inFn) 

        blkCountIn += 1

    }

    function readSlice(n) {
        let buffer = new Buffer(n)
        var readCount = fs.readSync(inF, buffer, null, n, offset)
        if(readCount < n) {
            console.log("END OF FILE. read count ",readCount, "<", n)
            return null
        }
        offset += n
        return buffer
    }
}

BlockDataCopier()

The weirdest thing is that if I uncomment line // if(inFn == 3) break then it works until it reaches file number 3 (blk00003.dat). It also work with 30 but not with 300. I have to process (right now) 748 files of around 250MB each.
This kind of weird situation is what I call a poltergeist.
When it hangs it never reaches this point so I am mystified by this.

Even whenif(inFn == 3) break is uncommented it freezes for an instant around block 350 and then it goes until condition inFn == 3 is met.

Any ideas?

Thanks.

1 Like