How to fix server crashing causing with the simple-imap extracting-body-data codes

Hello Geek,

I know it’s not the right place to post it here but since we are developing it under meteor we hope some people here can help us with this issue.

Our app on meteor-galaxy server keep crashing and we found out that the reason of it is our codes for our email cron job. We are using imap-simple npm package but since there’s no good tutorial on how to extract encoded data from email server, we added some lines to decode it…but seems like it stressing the server and server’s crashed. Below are our whole codes.

	var Imap    = require('imap');
        var base64  = require('base64-stream');
        var base64_loc  = require('base-64');
        var utf8 = require('utf8');
    
        var credentials = {accessKeyId: Meteor.settings.AWSAccessKeyId, secretAccessKey: Meteor.settings.AWSSecretAccessKey};
            
        AWS.config.update(credentials);
        AWS.config.region = Meteor.settings.AWSRegion;
        
        // create bucket instance
        var bucket = new AWS.S3({params: {Bucket: Meteor.settings.S3Bucket}});
    
    
        var imaps = require('imap-simple');
         
        var config = {
            imap: {
                user: Meteor.settings.IMAP_USERNAME,
                password: Meteor.settings.IMAP_PASSWORD,
                host: Meteor.settings.IMAP_HOST,
                port: Meteor.settings.IMAP_PORT,
                tls: true,
                authTimeout: 20000
            }
        };
        
        imaps.connect(config).then(function (connection2) {
         
            return connection2.openBox('INBOX').then(function () {
         
                // Fetch emails from the last 24h
                var delay = 24 * 3600 * 1000;
                var yesterday = new Date();
                yesterday.setTime(Date.now() - delay);
                yesterday = yesterday.toISOString();
                //var searchCriteria = ['UNSEEN', ['SINCE', yesterday]];
                var searchCriteria = ['UNSEEN'];
                //var fetchOptions = { bodies: ['HEADER', 'TEXT'], struct: true, markSeen: false };
                var fetchOptions = { bodies: ['HEADER', 'TEXT'], struct: true, markSeen: false };
         
                // retrieve only the headers of the messages
                return connection2.search(searchCriteria, fetchOptions);
            }).then(function (messages) {
                
               // console.log('Cron Emails Messaages is being read...');
                //console.log('Number of unread emails: ' + messages.length);
                
                messages.map(function (message) {
                        
                            try{
                            
                            let _check_exists = Promise.await(Emails_Reply.findOne({uid : message.attributes.uid}));
                            //console.log(_check_exists);
                            if(_check_exists){
                                //console.log('Email UID '+ message.attributes.uid +' is already exist.')
                                connection2.addFlags(message.attributes.uid, ['\\Seen'], function(err) {});
                                
                                return false;
                            }
                            
                            var attachments = [];
                            
                            let _header = message.parts.filter(function (part) {
                                    return part.which === 'HEADER'; 
                                })[0].body;
                                
                                
                            let email_subject = _header.subject[0];
                            
                            let _splt = email_subject.toLowerCase().split('#email');
                            if(typeof _splt[1] == 'undefined'){
                                return false;
                            }
                            _splt = (_splt[1]).split(' - ');
                            
                            let _email_number = 0;
                            
                            if(_splt[0] != '' && Number(_splt[0]) != 'NaN'){
                                _email_number = Number(_splt[0]);
                            }else{
                                return false;
                            }
                            
                            let emailInfo = Promise.await(Emails.findOne({_number: _email_number}));
                            
                            if(!emailInfo){
                                return false; 
                            }
                            
                            
                           let _body = message.parts.filter(function (part) {
                                return part.which === 'TEXT'; 
                            })[0].body; 
                                
                            //console.log(_body);
                            //return false;
                            
                            let _extract_body_first = (_body).split('<div class=3D"freshdesk_quote">');
                            if(_extract_body_first.length < 2){
                                let _extract_body = (_body).split('Content-Disposition: attachment;');
                                
                                _extract_body = (_extract_body[0]).split('Content-Type: text/html; charset=utf-8');
                                
                                if(typeof _extract_body[1] == 'undefined'){
                                    _extract_body = (_extract_body[0]).split('Content-Type: text/plain; charset="UTF-8"\r\n\r\n');
                                    _secure_spliter = _extract_body[0].split('Content-Transfer-Encoding: base64\r\n\r\n');
                                }else{
                                    _secure_spliter = _extract_body[1].split('Content-Transfer-Encoding: base64\r\n\r\n');
                                }
                                
                                if(_secure_spliter.length > 1){
                                  _secure_spliter = _secure_spliter[1].split('\r\n\r\n');
                                  
                                  _extract_body_2 = (base64_loc.decode(_secure_spliter[0])).split('-------- Original message --------');
                                  _extract_body[0] = _extract_body_2[0];
                                  email_text = utf8.decode(_extract_body[0]);
                                  
                                  //console.log(email_text);
                                  
                                }else{
                                    if(typeof _extract_body[1] != 'undefined')
                                        _extract_body = (_extract_body[1]).split('\r\n\r\n--');
                                    
                                    if(_extract_body.length < 2){
                                        _extract_body = (_extract_body[0]).split('\r\n\r\nOn ');
                                    }
                                    
                                    
                                    _extract_body = _extract_body[0].split('Content-Transfer-Encoding: quoted-printable\r\n\r\n');
                                    
                                    if(typeof _extract_body[1] != 'undefined'){
                                        _extract_body = (_extract_body[1]).split('Content-Type: text/html; charset=UTF-8');
                                        //console.log(_extract_body[0]);
                                        //if(typeof _extract_body[1] != 'undefined'){
                                        _extract_body = (_extract_body[0]).split('------ Original Message ------');
                                        //}
                                        _extract_body = _extract_body[0].split('\r\n\r\n');
                                        //console.log(_extract_body[0]);
                                        
                                        email_text = _extract_body[0].replace(new RegExp('=20\r\n', 'g'), ' ');
                                    }else{
                                        if(_extract_body.length > 1){
                                            email_text = _extract_body[1];
                                        }else{
                                            email_text = _extract_body[0];
                                        }
                                    }
                                    
                                    //console.log(email_text);
                                }
                            }else{
                                _extract_body_first = (_extract_body_first[0]).split('Content-Type: text/html;');
                                _extract_body_first = (_extract_body_first[1]).split('Content-Transfer-Encoding: quoted-printable\r\n\r\n');
                                email_text = _extract_body_first[1].replace('=C2=A0', '');
                                email_text = email_text.replace(new RegExp('3D', 'g'), '');
                                email_text = email_text.replace(new RegExp('<div', 'g'), '<span');
                                email_text = email_text.replace(new RegExp('</div', 'g'), '</span');
                                email_text = email_text.replace(new RegExp('<a', 'g'), '<l');
                                email_text = email_text.replace(new RegExp('</a', 'g'), '</l');
                                email_text = email_text.replace(new RegExp('=\r\n', 'g'), '');
                            }
                            
                            let _last_cleaner = email_text.split('\r\n\r\nOn');
                            if(typeof _last_cleaner[1] != 'undefined'){
                                email_text = _last_cleaner[0];
                            }
                            
                            let _subcleaner = email_text.split('<body>');
                            
                            if(typeof _subcleaner[1] != 'undefined'){
                                _subcleaner = _subcleaner[1].split('</body>');
                                email_text = _subcleaner[0];
                            }
                            
                            
                            let job_number = (typeof emailInfo['job_number'] != 'undefined') ? emailInfo['job_number'] : 0 ;
                            
                            let email_from = _header.from[0];
                            
                            
                            var parts = imaps.getParts(message.attributes.struct);
                            
                            let _attachments = parts.filter(function (part) {
                                return part.disposition && part.disposition.type.toUpperCase() === 'ATTACHMENT';
                            });
                            
                            let _attID = Promise.await(Emails_Reply.insert({
                                uid : message.attributes.uid
                            }));
                            
                            if(!_attID){
                                
                                return false;
                            }
                            
                            for(_i in _attachments){
                                part = _attachments[_i];
                                
                                attachments.push(Promise.await(connection2.getPartData(message, part)
                                    .then(function (partData) {
                                        let _fname_extract = (part.disposition.params.filename).split('.');
                                        let _fext = _fname_extract[_fname_extract.length - 1];
                                        let _new_fname = new Meteor.Collection.ObjectID().toHexString();
                                        
                                        fs.writeFileSync(tmp_path + '/' + _new_fname + '.' + _fext, partData, {encoding: 'binary'});
                                        
                                        stats = fs.statSync(tmp_path + '/' + _new_fname + '.' + _fext);
                                        
                                        
                                        if(stats.size < 6){ //less than 6kb size ignore
                                            fs.unlink(tmp_path + '/' + _new_fname + '.' + _fext);
                                            return false;
                                        }
                                        
                                        date = new Date();
                                          
                                      let _floc = "attachments/" + date.getFullYear() + '/' + (date.getMonth() + 1) + '/' + _new_fname + '.' + _fext;
                                      
                                      _resp_params = {
                                        orig_name : part.disposition.params.filename,
                                        filename : part.disposition.params.filename,
                                        new_name : _new_fname + '.' + _fext,
                                        description : part.disposition.params.filename,
                                        size : stats.size,
                                        deleted : false,
                                        full_url : Meteor.settings.S3BaseURL + _floc,
                                        path : Meteor.settings.S3BaseURL + _floc,
                                      };
                                        
                                        var params = {
                                              Key: _floc,
                                              Body: partData,
                                              ACL: 'public-read'
                                          };
                                          
                                        Promise.await(bucket.putObject(params, function (err, data) {
                                            if (err) {
                                                _resp_params = false;
                                            }else{
                                                fs.unlink(tmp_path + '/' + _new_fname + '.' + _fext);
                                            }
                                        }));
                                        
                                        return _resp_params;
                                    })));
                            }
                            
                            let tmp_attachments = [];
                            for(_it in attachments){
                                if(!attachments[_it]) continue;
                                tmp_attachments.push(attachments[_it]);
                            }
                            
                            Emails_Reply.update(_attID, {$set : {
                                parent_id : _email_number,
                                job_number : job_number,
                                from : email_from,
                                subject : email_subject,
                                message : email_text,
                                attached_files : tmp_attachments,
                                logged_date : new Date(),
                                is_read : false,
                                deleted : false,
                            }});
                            
                            _note = 'New replied email:<br/>';
                            
                            _note += 'Ref. ID: '+ _attID +'<br/><br/>';
                            
                            _note += 'Email Info:<br/>';
                            _note += 'From: ' + email_from + '<br/>';
                            _note += 'Subject: ' + email_subject + '<br/>';
                            _note += 'Text: ' + email_text + '<br/><br/>';
                            _note += 'Files Info: ' + JSON.stringify(tmp_attachments);
                            
                            let _n = {
                                job_number : job_number,
                                note : _note,
                                added_by : companyInfo.user_id,
                                type : (!job_number) ? 'email' : 'job',
                                show_on_customer : false,
                            };
                            
                            let _nID = Promise.await(Meteor.call('_addNote', _n, 1));
                            
                            connection2.addFlags(message.attributes.uid, ['\\Seen'], function(err) {
                                
                            });
                            
                        }catch(err){
                            console.log(err);
                        } 
                    
                });
                
                if(!messages.length){
                    connection2.end();
                }
                
                return true;
            });
        }).then(function(){
            //console.log('Email Cron Imap Connected');
        }).catch(function(err){
            //console.log('Email Cron Err:' + err);
            if(typeof connection2 != 'undefined') connection2.end();
        });;
    
    }catch(e){               
        //console.log('Email Cron Err: ' + e.message);
    }

And below are the cron codes.

if(Meteor.settings.ENABLE_EMAIL_CRON){
    new cron2.CronJob({
      cronTime: '*\/5 * * * *',
      onTick: function() {
        _jobEmailCron();
        return 1;
      },
      start: true,
      timeZone: 'Australia/Sydney'
    });
}

Thanks in advance for the possible help guys. :slight_smile:

Your issue is probably the IMAP search you’re invoking. You can write a test for your parsing code using various actual results and some random pieces of text, you will begin to understand what if anything is wrong with it.

Also, make sure to fully understand @robfallows post about how to use async / await: https://blog.meteor.com/using-promises-and-async-await-in-meteor-8f6f4a04f998

Thanks for the reply but I was using the Promise method before but still the same issue most of the base 64 encoded emails that the system extracted causing the server-crash.

Maybe isolate the base64 part and test it?