If you have ever spent any time dealing with folder watchers in pretty much any language, you will probably have noticed that the watcher usually notifies your application of a new file added to the folder the instant the file is added to the folder’s index. This however does not mean that the file is complete, it may still be in the process of being copied or saved to the disk. This creates a problem when you are watching for new files so that they can then be processed in some manner. To get around this you will need to check the API for the language/platform you are using to find out if the IO library has a way to check if a file is whole before processing.

I ran into this same issue while building a bulk video file transcoding system on top of the Node.js JavaScript platform. Unfortunately I could not find a built in method for checking if a file was completely saved before acting on it and so had to handle this myself. The following assumes you know how to use Chokidar or some other Node file watcher package from NPM. The function of interest is titled ‘checkFileCopyComplete’, hopefully this will help speed things up if you are looking for this solution.

// Setup video source folder observer for notifications of new files
var chokidar = require('chokidar');

var watcher = chokidar.watch(config.videoRawFolder, {
    persistent: true,
    followSymlinks: false,
    usePolling: true,
    depth: undefined,
    interval: 100,
    ignorePermissionErrors: false
});

watcher
    .on('ready', function() { logger.info('Initial scan complete. Ready for changes.'); })
    .on('unlink', function(path) { logger.info('File: ' + path + ', has been REMOVED'); })
    .on('error', function(err) {
        logger.error(component + 'Chokidar file watcher failed. ERR: ' + err.message);
    })
    .on('add', function(path) {
        logger.info('File', path, 'has been ADDED');

        fs.stat(path, function (err, stat) {

            if (err){
                logger.error(component + 'Error watching file for copy completion. ERR: ' + err.message);
                logger.error(component + 'Error file not processed. PATH: ' + path);
            } else {
                logger.info(component + 'File copy started...');
                setTimeout(checkFileCopyComplete, fileCopyDelaySeconds*1000, path, stat);
            }
        });
    });

// Makes sure that the file added to the directory, but may not have been completely copied yet by the
// Operating System, finishes being copied before it attempts to do anything with the file.
function checkFileCopyComplete(path, prev) {
    fs.stat(path, function (err, stat) {

        if (err) {
            throw err;
        }
        if (stat.mtime.getTime() === prev.mtime.getTime()) {
            logger.info(component + 'File copy complete => beginning processing');
            //------------------------------------- 
            // CALL A FUNCTION TO PROCESS FILE HERE
            //-------------------------------------
        }
        else {
            setTimeout(checkFileCopyComplete, fileCopyDelaySeconds*1000, path, stat);
        }
    });
}