Incremental Update based on a folder-hash file

package.json

@@ -13,6 +13,7 @@
         "minimatch": "3.0.4",
         "promise-ftp": "^1.3.5",
         "read": "^1.0.7",
+        "json-diff": "*",
         "ssh2-sftp-client": "^5.3.2",
         "upath": "^2.0.1"
     },

src/ftp-deploy.js

@@ -1,10 +1,11 @@
 "use strict";
 
+const fs = require("fs");
+const path = require("path");
 const upath = require("upath");
 const util = require("util");
 const events = require("events");
 const Promise = require("bluebird");
-const fs = require("fs");
 
 var PromiseFtp = require("promise-ftp");
 var PromiseSftp = require("ssh2-sftp-client");
@@ -23,28 +24,99 @@ const lib = require("./lib");
 const FtpDeployer = function () {
     // The constructor for the super class.
     events.EventEmitter.call(this);
+    this.transferFileMap = null,
+    this.deleteFileMap = null,
     this.ftp = null;
     this.eventObject = {
         totalFilesCount: 0,
         transferredFileCount: 0,
+        deletedFileCount: 0,
         filename: "",
     };
 
-    this.makeAllAndUpload = function (remoteDir, filemap) {
-        let keys = Object.keys(filemap);
+    this.execUpload = (config) => {
+        let keys = Object.keys(this.transferFileMap);
+        // check if has incremental hash filename - if yes put at last on upload list
+        let parked_fileFolderHashSums = false;
+        if (config.fileFolderHashSums && (config.fileFolderHashSums != '')) {
+            const check_filemap = this.transferFileMap['/'] || [];
+            for (let i = 0; i < check_filemap.length; i++) {
+                if (check_filemap[i] == config.fileFolderHashSums) {
+                    parked_fileFolderHashSums = true;
+                    check_filemap.splice(i, 1);
+                    this.transferFileMap['/'] = check_filemap;
+                    break;
+                }
+            }
+        }
+        // append special $ key if need for parked
+        if (parked_fileFolderHashSums) { keys.push('$'); }
+
         return Promise.mapSeries(keys, (key) => {
-            // console.log("Processing", key, filemap[key]);
-            return this.makeAndUpload(remoteDir, key, filemap[key]);
+            if (key == '$') {
+                return this.makeAndUpload(config, '/', [config.fileFolderHashSums]);
+            }
+            else {
+                return this.makeAndUpload(config, key, this.transferFileMap[key]);
+            }
         });
     };
 
-    this.makeDir = function (newDirectory) {
+    this.makeDir = (newDirectory) => {
         if (newDirectory === "/") {
             return Promise.resolve("unused");
         } else {
             return this.ftp.mkdir(newDirectory, true);
         }
     };
+
+    this.incrementalUpdate = (config, ftp_fileFolderHashSumsContent) => {
+        const diff_content = lib.getFolderHashSumsDiffs(
+                                 config.fileFolderHashSums,
+                                 config.localRoot,
+                                 ftp_fileFolderHashSumsContent
+                             );
+        if (diff_content) {
+            config.include = diff_content.upload;
+            config.delete = diff_content.delete;
+        }
+        return config
+    }
+
+    // Creates a remote directory and uploads all of the files in it
+    // Resolves a confirmation message on success
+    this.getRemoteHashFile = (config) => {
+        if (!config.fileFolderHashSums || (config.fileFolderHashSums == "") || !config.deleteRemote) {
+            return config;
+        } else {
+            const fname = path.posix.join(config.remoteRoot, config.fileFolderHashSums);
+            return this.ftp.get(fname).then(stream => {
+                // create a new reader promise
+                return new Promise(function (resolve, reject) {
+                    // buffer reader
+                    stream.on('readable', () => {
+                        let buf = stream.read(1024*1024*10);
+                        if (buf != null) {
+                            // watch dog timer to close connection while file was loaded
+                            setTimeout(() => { stream.destroy(); }, 1000);
+                            // send the readed buf in once to .then
+                            resolve("" + buf);
+                        }
+                    });
+                })
+            })
+            .then((content) => {
+                return this.incrementalUpdate(config, content);
+            })
+            .catch((err) => {
+                // we need to update all
+                config.include = ["**/*"]
+                config.delete = ["/"]
+                return config;
+            })
+        }
+    }
+
     // Creates a remote directory and uploads all of the files in it
     // Resolves a confirmation message on success
     this.makeAndUpload = (config, relDir, fnames) => {
@@ -54,7 +126,7 @@ const FtpDeployer = function () {
             return Promise.mapSeries(fnames, (fname) => {
                 let tmpFileName = upath.join(config.localRoot, relDir, fname);
                 let tmp = fs.readFileSync(tmpFileName);
-                this.eventObject["filename"] = upath.join(relDir, fname);
+                this.eventObject["filename"] = upath.join(config.remoteRoot, relDir, fname);
 
                 this.emit("uploading", this.eventObject);
 
@@ -122,35 +194,88 @@ const FtpDeployer = function () {
     };
 
     // creates list of all files to upload and starts upload process
-    this.checkLocalAndUpload = (config) => {
+    this.checkLocal = (config) => {
         try {
-            let filemap = lib.parseLocal(
+            this.transferFileMap = lib.parseLocal(
                 config.include,
                 config.exclude,
                 config.localRoot,
                 "/"
             );
-            // console.log(filemap);
+
+            // console.log(this.transferFileMap);
             this.emit(
                 "log",
-                "Files found to upload: " + JSON.stringify(filemap)
+                "Files found to upload: " + JSON.stringify(this.transferFileMap)
             );
-            this.eventObject["totalFilesCount"] = lib.countFiles(filemap);
+            this.eventObject["totalFilesCount"] += lib.countFiles(this.transferFileMap);
 
-            return this.makeAllAndUpload(config, filemap);
+            return Promise.resolve(config);
         } catch (e) {
             return Promise.reject(e);
         }
     };
 
+    this.execDeleteRemotes= (deletes, remoteRootDir='') => {
+        return Promise.mapSeries(deletes, item => {
+            // use parent dir to locate search mask
+            let dir = path.posix.join(remoteRootDir, item.split('/').slice(0,-1).join('/'));
+            let fmask = item.split('/').slice(-1);
+
+            return this.ftp.list(dir).then(lst => {
+
+                let dirNames = lst
+                    .filter(f => f.type == "d" && f.name != ".." && f.name != "." && ((fmask == '') || (f.name == fmask)))
+                    .map(f => path.posix.join(dir, f.name));
+
+                let fnames = lst
+                    .filter(f => (f.type != "d" && ((fmask == '') || (f.name == fmask))))
+                    .map(f => path.posix.join(dir, f.name));
+
+                this.deleteFileMap = this.deleteFileMap.concat(dirNames, fnames);
+                this.eventObject["totalFilesCount"] += dirNames.length + fnames.length;
+
+                // delete sub-directories and then all files
+                return Promise.mapSeries(dirNames, dirName => {
+                    // deletes everything in sub-directory, and then itself
+                    return this
+                        .execDeleteRemotes([dirName + '/'])
+                        .then(() => this.ftp.rmdir(dirName)
+                            .then(() => {
+                              this.eventObject.deletedFileCount++;
+                              this.eventObject["filename"] = dirName;
+                              this.emit("removed", this.eventObject);
+                            })
+                        );
+                })
+                    .then(() =>
+                        Promise.mapSeries(fnames, fname =>
+                            this.ftp.delete(fname)
+                                .then(() => {
+                                    this.eventObject.deletedFileCount++;
+                                    this.eventObject["filename"] = fname;
+                                    this.emit("removed", this.eventObject);
+                                })
+                        )
+                    );
+            })
+        });
+    }
+
     // Deletes remote directory if requested by config
     // Returns config
     this.deleteRemote = (config) => {
         if (config.deleteRemote) {
-            return lib
-                .deleteDir(this.ftp, config.remoteRoot)
-                .then(() => {
-                    this.emit("log", "Deleted directory: " + config.remoteRoot);
+            this.deleteFileMap = [];
+            let filemap = lib.parseDeletes(
+                config.delete,
+                config.remoteRoot,
+                (config.fileFolderHashSums && (config.fileFolderHashSums != ''))
+            );
+            return this
+                .execDeleteRemotes(filemap, config.remoteRoot)
+                .then((done) => {
+                    this.emit("log", "Deleted remotes: " + JSON.stringify(done));
                     return config;
                 })
                 .catch((err) => {
@@ -170,14 +295,24 @@ const FtpDeployer = function () {
             .checkIncludes(config)
             .then(lib.getPassword)
             .then(this.connect)
+            .then(this.getRemoteHashFile)
+            .then(this.checkLocal)
             .then(this.deleteRemote)
-            .then(this.checkLocalAndUpload)
+            .then(this.execUpload)
             .then((res) => {
                 this.ftp.end();
+                const data = {
+                  totalFilesCount: this.eventObject.totalFilesCount,
+                  transferredFileCount: this.eventObject.transferredFileCount,
+                  deletedFileCount: this.eventObject.deletedFileCount,
+                  transferFileMap: this.transferFileMap,
+                  deleteFileMap: this.deleteFileMap,
+                  res: res,
+                }
                 if (typeof cb == "function") {
-                    cb(null, res);
+                    cb(null, data);
                 } else {
-                    return Promise.resolve(res);
+                    return Promise.resolve(data);
                 }
             })
             .catch((err) => {