From 95dc275ad9470f3f35ba4f05f46a762bcf400005 Mon Sep 17 00:00:00 2001 From: Tom Freudenberg Date: Sat, 31 Oct 2020 14:44:51 +0100 Subject: [PATCH 1/6] add: allow deleting of defined files and folders befor upload --- package.json | 2 +- src/ftp-deploy.js | 59 +++++++++++++++++++++++++++++++++++++++++++---- src/lib.js | 31 +++++++++---------------- 3 files changed, 66 insertions(+), 26 deletions(-) diff --git a/package.json b/package.json index f2ce253..21dd053 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "devDependencies": { "chai": "^4.2.0", "delete": "^1.1.0", - "ftp-srv": "^4.2.0", + "ftp-srv": "^4.3.0", "mocha": "^6.2.1" }, "keywords": [ diff --git a/src/ftp-deploy.js b/src/ftp-deploy.js index 98afd12..b7829ba 100644 --- a/src/ftp-deploy.js +++ b/src/ftp-deploy.js @@ -1,5 +1,6 @@ "use strict"; +const path = require("path"); const upath = require("upath"); const util = require("util"); const events = require("events"); @@ -9,7 +10,7 @@ const fs = require("fs"); var PromiseFtp = require("promise-ftp"); const lib = require("./lib"); -/* interim structure +/* interim structure { '/': ['test-inside-root.txt'], 'folderA': ['test-inside-a.txt'], @@ -26,6 +27,7 @@ const FtpDeployer = function () { this.eventObject = { totalFilesCount: 0, transferredFileCount: 0, + deletedFileCount: 0, filename: "", }; @@ -116,14 +118,61 @@ const FtpDeployer = function () { } }; + this.execDeleteRemotes= (deletes, remoteRootDir='') => { + return Promise.mapSeries(deletes, item => { + // use parent dir to locate search mask + let dir = path.posix.join(remoteRootDir, item.split('/').slice(0,-1).join('/')); + let fmask = item.split('/').slice(-1); + + return this.ftp.list(dir).then(lst => { + + let dirNames = lst + .filter(f => f.type == "d" && f.name != ".." && f.name != "." && ((fmask == '') || (f.name == fmask))) + .map(f => path.posix.join(dir, f.name)); + + let fnames = lst + .filter(f => (f.type != "d" && ((fmask == '') || (f.name == fmask)))) + .map(f => path.posix.join(dir, f.name)); + + // delete sub-directories and then all files + return Promise.mapSeries(dirNames, dirName => { + // deletes everything in sub-directory, and then itself + return this + .execDeleteRemotes([dirName + '/']) + .then(() => this.ftp.rmdir(dirName) + .then(() => { + this.eventObject.deletedFileCount++; + this.eventObject["filename"] = dirName; + this.emit("removed", this.eventObject); + }) + ); + }) + .then(() => + Promise.mapSeries(fnames, fname => + this.ftp.delete(fname) + .then(() => { + this.eventObject.deletedFileCount++; + this.eventObject["filename"] = fname; + this.emit("removed", this.eventObject); + }) + ) + ); + }) + }); + } + // Deletes remote directory if requested by config // Returns config this.deleteRemote = (config) => { if (config.deleteRemote) { - return lib - .deleteDir(this.ftp, config.remoteRoot) - .then(() => { - this.emit("log", "Deleted directory: " + config.remoteRoot); + let filemap = lib.parseDeletes( + config.delete, + config.remoteRoot + ); + return this + .execDeleteRemotes(filemap, config.remoteRoot) + .then((done) => { + this.emit("log", "Deleted remotes: " + JSON.stringify(filemap)); return config; }) .catch((err) => { diff --git a/src/lib.js b/src/lib.js index 8d66eb1..de5da2b 100644 --- a/src/lib.js +++ b/src/lib.js @@ -106,29 +106,20 @@ function parseLocal(includes, excludes, localRootDir, relDir) { return res; } +function parseDeletes(deletes, remoteRootDir) { + const res = []; + deletes = deletes || []; + deletes.forEach(function(item) { + res.push(item.slice(-1) == '/' ? item.slice(0,-1) : item); + }); + return res; +} + function countFiles(filemap) { return Object.values(filemap).reduce((acc, item) => acc.concat(item)) .length; } -function deleteDir(ftp, dir) { - return ftp.list(dir).then(lst => { - let dirNames = lst - .filter(f => f.type == "d" && f.name != ".." && f.name != ".") - .map(f => path.posix.join(dir, f.name)); - - let fnames = lst - .filter(f => f.type != "d") - .map(f => path.posix.join(dir, f.name)); - - // delete sub-directories and then all files - return Promise.mapSeries(dirNames, dirName => { - // deletes everything in sub-directory, and then itself - return deleteDir(ftp, dirName).then(() => ftp.rmdir(dirName)); - }).then(() => Promise.mapSeries(fnames, fname => ftp.delete(fname))); - }); -} - mkDirExists = (ftp, dir) => { // Make the directory using recursive expand return ftp.mkdir(dir, true).catch(err => { @@ -146,8 +137,8 @@ module.exports = { checkIncludes: checkIncludes, getPassword: getPassword, parseLocal: parseLocal, + parseDeletes: parseDeletes, canIncludePath: canIncludePath, countFiles: countFiles, - mkDirExists: mkDirExists, - deleteDir: deleteDir + mkDirExists: mkDirExists }; From 1e3b44b1765242d74017afeb37fdf66c3349bf66 Mon Sep 17 00:00:00 2001 From: Tom Freudenberg Date: Sat, 31 Oct 2020 14:50:06 +0100 Subject: [PATCH 2/6] fix: if empty delete, remove completely remote folders (as previous default) --- src/lib.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib.js b/src/lib.js index de5da2b..0d60885 100644 --- a/src/lib.js +++ b/src/lib.js @@ -112,6 +112,9 @@ function parseDeletes(deletes, remoteRootDir) { deletes.forEach(function(item) { res.push(item.slice(-1) == '/' ? item.slice(0,-1) : item); }); + if (res.length == 0) { + res.push('/'); + } return res; } From 4421417fc34cfa8ee0d8e4ac43a27761620e6719 Mon Sep 17 00:00:00 2001 From: Tom Freudenberg Date: Sat, 31 Oct 2020 14:50:53 +0100 Subject: [PATCH 3/6] fix: emit full path and filename to event emitter --- src/ftp-deploy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ftp-deploy.js b/src/ftp-deploy.js index b7829ba..f23292b 100644 --- a/src/ftp-deploy.js +++ b/src/ftp-deploy.js @@ -55,7 +55,7 @@ const FtpDeployer = function () { return Promise.mapSeries(fnames, (fname) => { let tmpFileName = upath.join(config.localRoot, relDir, fname); let tmp = fs.readFileSync(tmpFileName); - this.eventObject["filename"] = upath.join(relDir, fname); + this.eventObject["filename"] = upath.join(config.remoteRoot, fname); this.emit("uploading", this.eventObject); From d832a183c4d67674e33066a4fb57df6f4c2e3e7d Mon Sep 17 00:00:00 2001 From: Tom Freudenberg Date: Sat, 31 Oct 2020 15:45:10 +0100 Subject: [PATCH 4/6] fix: split calcing upload files and uploading, additional list for deleteFileMap --- src/ftp-deploy.js | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/ftp-deploy.js b/src/ftp-deploy.js index f23292b..394bf6d 100644 --- a/src/ftp-deploy.js +++ b/src/ftp-deploy.js @@ -23,6 +23,8 @@ const lib = require("./lib"); const FtpDeployer = function () { // The constructor for the super class. events.EventEmitter.call(this); + this.transferFileMap = null, + this.deleteFileMap = null, this.ftp = null; this.eventObject = { totalFilesCount: 0, @@ -31,11 +33,11 @@ const FtpDeployer = function () { filename: "", }; - this.makeAllAndUpload = function (remoteDir, filemap) { - let keys = Object.keys(filemap); + this.execUpload = (config) => { + let keys = Object.keys(this.transferFileMap); return Promise.mapSeries(keys, (key) => { // console.log("Processing", key, filemap[key]); - return this.makeAndUpload(remoteDir, key, filemap[key]); + return this.makeAndUpload(config, key, this.transferFileMap[key]); }); }; @@ -48,6 +50,7 @@ const FtpDeployer = function () { }; // Creates a remote directory and uploads all of the files in it // Resolves a confirmation message on success + this.makeAndUpload = (config, relDir, fnames) => { let newDirectory = upath.join(config.remoteRoot, relDir); return this.makeDir(newDirectory, true).then(() => { @@ -97,22 +100,22 @@ const FtpDeployer = function () { }; // creates list of all files to upload and starts upload process - this.checkLocalAndUpload = (config) => { + this.checkLocal = (config) => { try { - let filemap = lib.parseLocal( + this.transferFileMap = lib.parseLocal( config.include, config.exclude, config.localRoot, "/" ); - // console.log(filemap); + // console.log(this.transferFileMap); this.emit( "log", - "Files found to upload: " + JSON.stringify(filemap) + "Files found to upload: " + JSON.stringify(this.transferFileMap) ); - this.eventObject["totalFilesCount"] = lib.countFiles(filemap); + this.eventObject["totalFilesCount"] += lib.countFiles(this.transferFileMap); - return this.makeAllAndUpload(config, filemap); + return Promise.resolve(config); } catch (e) { return Promise.reject(e); } @@ -134,6 +137,9 @@ const FtpDeployer = function () { .filter(f => (f.type != "d" && ((fmask == '') || (f.name == fmask)))) .map(f => path.posix.join(dir, f.name)); + this.deleteFileMap = this.deleteFileMap.concat(dirNames, fnames); + this.eventObject["totalFilesCount"] += dirNames.length + fnames.length; + // delete sub-directories and then all files return Promise.mapSeries(dirNames, dirName => { // deletes everything in sub-directory, and then itself @@ -165,6 +171,7 @@ const FtpDeployer = function () { // Returns config this.deleteRemote = (config) => { if (config.deleteRemote) { + this.deleteFileMap = []; let filemap = lib.parseDeletes( config.delete, config.remoteRoot @@ -172,7 +179,7 @@ const FtpDeployer = function () { return this .execDeleteRemotes(filemap, config.remoteRoot) .then((done) => { - this.emit("log", "Deleted remotes: " + JSON.stringify(filemap)); + this.emit("log", "Deleted remotes: " + JSON.stringify(done)); return config; }) .catch((err) => { @@ -191,15 +198,24 @@ const FtpDeployer = function () { return lib .checkIncludes(config) .then(lib.getPassword) + .then(this.checkLocal) .then(this.connect) .then(this.deleteRemote) - .then(this.checkLocalAndUpload) + .then(this.execUpload) .then((res) => { this.ftp.end(); + const data = { + totalFilesCount: this.eventObject.totalFilesCount, + transferredFileCount: this.eventObject.transferredFileCount, + deletedFileCount: this.eventObject.deletedFileCount, + transferFileMap: this.transferFileMap, + deleteFileMap: this.deleteFileMap, + res: res, + } if (typeof cb == "function") { - cb(null, res); + cb(null, data); } else { - return Promise.resolve(res); + return Promise.resolve(data); } }) .catch((err) => { From 13cb86b52070c45985fee2346b0b5ed988ca00ac Mon Sep 17 00:00:00 2001 From: Tom Freudenberg Date: Sat, 31 Oct 2020 23:53:32 +0100 Subject: [PATCH 5/6] mod: finalize incremental update of ftp folders --- package.json | 3 +- src/ftp-deploy.js | 84 +++++++++++++++++++++++--- src/lib.js | 148 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 223 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 21dd053..74365ed 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "minimatch": "3.0.4", "promise-ftp": "^1.3.5", "read": "^1.0.7", - "upath": "^1.2.0" + "upath": "^1.2.0", + "json-diff": "*" }, "devDependencies": { "chai": "^4.2.0", diff --git a/src/ftp-deploy.js b/src/ftp-deploy.js index 394bf6d..8579c26 100644 --- a/src/ftp-deploy.js +++ b/src/ftp-deploy.js @@ -1,11 +1,11 @@ "use strict"; +const fs = require("fs"); const path = require("path"); const upath = require("upath"); const util = require("util"); const events = require("events"); const Promise = require("bluebird"); -const fs = require("fs"); var PromiseFtp = require("promise-ftp"); const lib = require("./lib"); @@ -35,22 +35,89 @@ const FtpDeployer = function () { this.execUpload = (config) => { let keys = Object.keys(this.transferFileMap); + // check if has incremental hash filename - if yes put at last on upload list + let parked_fileFolderHashSums = false; + if (config.fileFolderHashSums && (config.fileFolderHashSums != '')) { + const check_filemap = this.transferFileMap['/'] || []; + for (let i = 0; i < check_filemap.length; i++) { + if (check_filemap[i] == config.fileFolderHashSums) { + parked_fileFolderHashSums = true; + check_filemap.splice(i, 1); + this.transferFileMap['/'] = check_filemap; + break; + } + } + } + // append special $ key if need for parked + if (parked_fileFolderHashSums) { keys.push('$'); } + return Promise.mapSeries(keys, (key) => { - // console.log("Processing", key, filemap[key]); - return this.makeAndUpload(config, key, this.transferFileMap[key]); + if (key == '$') { + return this.makeAndUpload(config, '/', [config.fileFolderHashSums]); + } + else { + return this.makeAndUpload(config, key, this.transferFileMap[key]); + } }); }; - this.makeDir = function (newDirectory) { + this.makeDir = (newDirectory) => { if (newDirectory === "/") { return Promise.resolve("unused"); } else { return this.ftp.mkdir(newDirectory, true); } }; + + this.incrementalUpdate = (config, ftp_fileFolderHashSumsContent) => { + const diff_content = lib.getFolderHashSumsDiffs( + config.fileFolderHashSums, + config.localRoot, + ftp_fileFolderHashSumsContent + ); + if (diff_content) { + config.include = diff_content.upload; + config.delete = diff_content.delete; + } + return config + } + // Creates a remote directory and uploads all of the files in it // Resolves a confirmation message on success + this.getRemoteHashFile = (config) => { + if (!config.fileFolderHashSums || (config.fileFolderHashSums == "") || !config.deleteRemote) { + return config; + } else { + const fname = path.posix.join(config.remoteRoot, config.fileFolderHashSums); + return this.ftp.get(fname).then(stream => { + // create a new reader promise + return new Promise(function (resolve, reject) { + // buffer reader + stream.on('readable', () => { + let buf = stream.read(1024*1024*10); + if (buf != null) { + // watch dog timer to close connection while file was loaded + setTimeout(() => { stream.destroy(); }, 1000); + // send the readed buf in once to .then + resolve("" + buf); + } + }); + }) + }) + .then((content) => { + return this.incrementalUpdate(config, content); + }) + .catch((err) => { + // we need to update all + config.include = ["**/*"] + config.delete = ["/"] + return config; + }) + } + } + // Creates a remote directory and uploads all of the files in it + // Resolves a confirmation message on success this.makeAndUpload = (config, relDir, fnames) => { let newDirectory = upath.join(config.remoteRoot, relDir); return this.makeDir(newDirectory, true).then(() => { @@ -58,7 +125,7 @@ const FtpDeployer = function () { return Promise.mapSeries(fnames, (fname) => { let tmpFileName = upath.join(config.localRoot, relDir, fname); let tmp = fs.readFileSync(tmpFileName); - this.eventObject["filename"] = upath.join(config.remoteRoot, fname); + this.eventObject["filename"] = upath.join(config.remoteRoot, relDir, fname); this.emit("uploading", this.eventObject); @@ -108,6 +175,7 @@ const FtpDeployer = function () { config.localRoot, "/" ); + // console.log(this.transferFileMap); this.emit( "log", @@ -174,7 +242,8 @@ const FtpDeployer = function () { this.deleteFileMap = []; let filemap = lib.parseDeletes( config.delete, - config.remoteRoot + config.remoteRoot, + (config.fileFolderHashSums && (config.fileFolderHashSums != '')) ); return this .execDeleteRemotes(filemap, config.remoteRoot) @@ -198,8 +267,9 @@ const FtpDeployer = function () { return lib .checkIncludes(config) .then(lib.getPassword) - .then(this.checkLocal) .then(this.connect) + .then(this.getRemoteHashFile) + .then(this.checkLocal) .then(this.deleteRemote) .then(this.execUpload) .then((res) => { diff --git a/src/lib.js b/src/lib.js index 0d60885..3cd6c5c 100644 --- a/src/lib.js +++ b/src/lib.js @@ -8,11 +8,14 @@ const readP = util.promisify(read); const minimatch = require("minimatch"); +const jsonDiff = require("json-diff"); + + // P H A S E 0 function checkIncludes(config) { config.excludes = config.excludes || []; - if (!config.include || !config.include.length) { + if ((!config.include || !config.include.length) && (!config.deleteRemote && !config.fileFolderHashSums && (config.fileFolderHashSums == '' ))) { return Promise.reject({ code: "NoIncludes", message: "You need to specify files to upload - e.g. ['*', '**/*']" @@ -106,13 +109,13 @@ function parseLocal(includes, excludes, localRootDir, relDir) { return res; } -function parseDeletes(deletes, remoteRootDir) { +function parseDeletes(deletes, remoteRootDir, incrementalUpdate) { const res = []; deletes = deletes || []; deletes.forEach(function(item) { res.push(item.slice(-1) == '/' ? item.slice(0,-1) : item); }); - if (res.length == 0) { + if ((res.length == 0) && !incrementalUpdate) { res.push('/'); } return res; @@ -136,6 +139,142 @@ mkDirExists = (ftp, dir) => { }); }; +// re-engineer content from folder-hash to object +function fh_folder_hash_to_obj(content) { + let content_json = String(content). + replace(/\'/g, '"'). + replace(/\" \}$/gm, '" },'). + replace(/\},\s*\]/gm, '} ]'). + replace(/\}\s*\{/gm, '}, {'). + replace(/name:/g, '"name":'). + replace(/hash:/g, '"hash":'). + replace(/children:/g, '"children":'); + return JSON.parse(content_json); +} + +// read a file generated from folder-hash and re-egineer to object +function fh_read_folder_hash(filename) { + let content = fs.readFileSync(filename); + return fh_folder_hash_to_obj(content); +} + +// rebuild a parent / children structured list from folder-hash +// to a flat path / filename list with hash +function fh_make_folder_list(obj, path, items) { + items.forEach(function(elem) { + path_and_name = path + (path != '' ? '/' : '') + elem.name; + obj.push({ hash: elem.hash, name: path_and_name + (elem.children ? '/' : '') }); + if (elem.children) { fh_make_folder_list(obj, path_and_name, elem.children); } + }); + return obj; +} + +// get the filename (not path) for a hash from a flat list +function fh_get_changed_filename_for_hash(hash_list, search_hash) { + // set a default + let result = '' + // iterate over list + hash_list.forEach(function(item) { + // check for searched hash + if (item.hash == search_hash) { + // return the filename if it is not a path + result = (item.name.slice(-1) != '/') ? item.name : ''; + // stop immediately this iteration + return; + } + }); + // return what and if found + return result; +} + +// drop all empty entries and if optimize for delete +// remove all subfiles when a folder is deleted, in case +// of NOT for delete, drop any path to upload just filenames +function fh_optimize_filelist(filelist, flag_optimize_for_delete) { + // optimize delete list if + let i = 0; + let rm_path = ''; + while (i < filelist.length) { + if (filelist[i] == '') { + filelist.splice(i, 1); + } + else + if (filelist[i].slice(-1) == '/') { + if (flag_optimize_for_delete) { + rm_path = filelist[i]; + i++; + } else { + filelist.splice(i, 1); + } + } + else + if ((rm_path != '') && filelist[i].startsWith(rm_path)) { + filelist.splice(i, 1); + } + else { + rm_path = ''; + i++; + } + } + // return the changed array + return filelist; +} + +// use jsonDiff and extract the filenames which are changed +function fh_get_diff_filelist(hashlist_a, hashlist_b) { + let diff = jsonDiff.diff(hashlist_a, hashlist_b) || []; + let result = []; + diff.forEach(function(elem) { + if (elem[0] == '-') { + result.push(elem[1].name) + } else + if (elem[0] == '~') { + result.push(fh_get_changed_filename_for_hash(hashlist_a, elem[1].hash.__old)); + } + }); + + return result; +} + +// prepare the list to delete on ftp and to upload +function fh_get_ftp_filelists(fileFolderHashSums, hashes_on_ftp, hashes_in_site) { + + result = { + delete: fh_optimize_filelist(fh_get_diff_filelist(hashes_on_ftp, hashes_in_site), true), + upload: fh_optimize_filelist(fh_get_diff_filelist(hashes_in_site, hashes_on_ftp), false) + } + // append ourself if any changes + if ((result.delete.length > 0) || (result.upload.length > 0)) { + result.delete.splice(0, 0, fileFolderHashSums) + result.upload.push(fileFolderHashSums) + } + + // console.log('\n\n-- DIFF RESULT -------------------\n\n'); + // console.log(result); + // console.log('\n\n---------------------\n\n'); + + // return result + return result; +} + +function getFolderHashSumsDiffs(fileFolderHashSums, localRoot, ftp_content) { + let res = null; + try { + const fname = path.join(localRoot, fileFolderHashSums) + if (fs.existsSync(fname)) { + res = fh_get_ftp_filelists( + fileFolderHashSums, + fh_make_folder_list([], '', fh_folder_hash_to_obj(ftp_content).children), + fh_make_folder_list([], '', fh_read_folder_hash(fname).children) + ); + } + } catch(err) { + } + + // return the prepared diff lists + return res; +} + module.exports = { checkIncludes: checkIncludes, getPassword: getPassword, @@ -143,5 +282,6 @@ module.exports = { parseDeletes: parseDeletes, canIncludePath: canIncludePath, countFiles: countFiles, - mkDirExists: mkDirExists + mkDirExists: mkDirExists, + getFolderHashSumsDiffs: getFolderHashSumsDiffs }; From 25fd20d154e4fd1c7fcc6dc5fa57f8d68ead4358 Mon Sep 17 00:00:00 2001 From: Tom Freudenberg Date: Sun, 1 Nov 2020 00:36:15 +0100 Subject: [PATCH 6/6] fix: if local folder-hash-sums missing switch to fallback and transfer all --- src/lib.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/lib.js b/src/lib.js index 3cd6c5c..ed88bc6 100644 --- a/src/lib.js +++ b/src/lib.js @@ -259,16 +259,15 @@ function fh_get_ftp_filelists(fileFolderHashSums, hashes_on_ftp, hashes_in_site) function getFolderHashSumsDiffs(fileFolderHashSums, localRoot, ftp_content) { let res = null; - try { - const fname = path.join(localRoot, fileFolderHashSums) - if (fs.existsSync(fname)) { - res = fh_get_ftp_filelists( - fileFolderHashSums, - fh_make_folder_list([], '', fh_folder_hash_to_obj(ftp_content).children), - fh_make_folder_list([], '', fh_read_folder_hash(fname).children) - ); - } - } catch(err) { + const fname = path.join(localRoot, fileFolderHashSums) + if (fs.existsSync(fname)) { + res = fh_get_ftp_filelists( + fileFolderHashSums, + fh_make_folder_list([], '', fh_folder_hash_to_obj(ftp_content).children), + fh_make_folder_list([], '', fh_read_folder_hash(fname).children) + ); + } else { + throw new Error("Cannot state the local file: " + fileFolderHashSums + " - fallback to full update!"); } // return the prepared diff lists