From 065a066f2c2f16421e19f6e237e0db98ccbad011 Mon Sep 17 00:00:00 2001 From: Martii Date: Sun, 23 Sep 2018 07:14:26 -0600 Subject: [PATCH] A dep change * Remove *fakes3* dep and all associated Ruby/Rails for pure JavaScript implementation * New dev dep *S3rver* handles what we need with basic i/o for S3 on dev. Please read their homepage * Leaving *fakeS3* `.gitignore`'s in for current data path... please delete your copies as this should eventually go away... or you can manually migrate it yourself perhaps. I'll just reimport from GH the sources on dev which is a better use of my time. * Fix a potential `undefined` I put in trap a while back with a generic string. * Change all docs to accommodate this * Moved a native dependency up top (the TODO)... code style conformance * DELETE OP RETESTED... PASS NOTE: * Contributors, and up, should no longer need to launch the server as it's launched in dev automatically and should terminate with a SIGINT (app exit). Remote local hosts still supported although I've never used that myself. Closes #1519 --- .github/CONTRIBUTING.md | 2 - .gitignore | 2 + Gemfile | 2 - README.md | 19 ++---- controllers/scriptStorage.js | 48 +++++++++++++- controllers/user.js | 2 +- dev/postinstall.js | 117 ----------------------------------- package.json | 3 + 8 files changed, 57 insertions(+), 138 deletions(-) delete mode 100644 Gemfile diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index c58cffd8c..7cfcfc793 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -10,8 +10,6 @@ This project uses [editor config](http://editorconfig.org/), please make sure to * [Git](http://git-scm.com/) * [node.js](http://nodejs.org/) *(see [`./package.json`](https://github.com/OpenUserJs/OpenUserJS.org/blob/master/package.json) engines for specific requirements)* * [MongoDB](http://www.mongodb.org/) (Optional. The project is preconfigured to use a dev DB on [MongoLab](https://mongolab.com/).) -* [Ruby](https://www.ruby-lang.org/) (required to run [FakeS3](https://github.com/jubos/fake-s3/)) -* [FakeS3](https://github.com/jubos/fake-s3) (required to store libraries/scripts without [AWS S3](http://aws.amazon.com/s3/)) handled by [bundler](https://github.com/bundler/bundler) #### GitHub Fork Setup diff --git a/.gitignore b/.gitignore index 3cf6ab945..d168ab0dd 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ data package-lock.json Gemfile.lock +S3rver + fakeS3 dev/fakeS3 diff --git a/Gemfile b/Gemfile deleted file mode 100644 index 3612774eb..000000000 --- a/Gemfile +++ /dev/null @@ -1,2 +0,0 @@ -source 'https://rubygems.org' -gem 'fakes3', '2.0.0' diff --git a/README.md b/README.md index da5cb0544..76cca364e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ Repository | Reference | Recent Version :--- | :---: | :--- [nodejs][nodeGHUrl] | [Documentation][nodejsDOCUrl] | Current release schedule [CHANGELOG][nodejsReleasesUrl] [npm][npmGHUrl] | [Documentation][npmDOCUrl] | [![npm][npmNPMVersionImage]][npmNPMUrl] [CHANGELOG][npmGHReleasesUrl] -[bundler][bundlerGHUrl] | [Documentation][bundlerDOCUrl] | [![GEM version][bundlerGEMVersionImage]][bundlerGEMUrl] [CHANGELOG][bundlerGHReleasesUrl] ### Contributing @@ -38,7 +37,6 @@ Repository | Reference | Recent Version [express-brute-mongo][express-brute-mongoGHUrl]
⋔ [`MongoDBv3.x`][express-brute-mongoGHMongoDBv3.xUrl] | [Documentation][express-brute-mongoDOCUrl] | [![NPM version][express-brute-mongoNPMVersionImage]][express-brute-mongoNPMUrl] [express-minify][express-minifyGHUrl] | [Documentation][express-minifyDOCUrl] | [![NPM version][express-minifyNPMVersionImage]][express-minifyNPMUrl] [express-session][express-sessionGHUrl] | [Documentation][express-sessionDOCUrl] | [![NPM version][express-sessionNPMVersionImage]][express-sessionNPMUrl] -[fakes3][fakes3GHUrl] | [Documentation][fakes3DOCUrl] | [![GEM version][fakes3GEMVersionImage]][fakes3GEMUrl] [font-awesome][font-awesomeGHUrl] | [Documentation][font-awesomeDOCUrl] | [![NPM version][font-awesomeNPMVersionImage]][font-awesomeNPMUrl] [formidable][formidableGHUrl] | [Documentation][formidableDOCUrl] | [![NPM version][formidableNPMVersionImage]][formidableNPMUrl] [git-rev][git-revGHUrl] | [Documentation][git-revDOCUrl] | [![NPM version][git-revNPMVersionImage]][git-revNPMUrl] @@ -75,6 +73,7 @@ Repository | Reference | Recent Version [pegjs][pegjsGHUrl] | [Documentation][pegjsDOCUrl] | [![NPM version][pegjsNPMVersionImage]][pegjsNPMUrl] [request][requestGHUrl] | [Documentation][requestDOCUrl] | [![NPM version][requestNPMVersionImage]][requestNPMUrl] [rfc2047][rfc2047GHUrl] | [Documentation][rfc2047DOCUrl] | [![NPM version][rfc2047NPMVersionImage]][rfc2047NPMUrl] +[S3rver][s3rverGHUrl] | [Documentation][s3rverDOCUrl] | [![NPM version][s3rverNPMVersionImage]][s3rverNPMUrl] [sanitize-html][sanitize-htmlGHUrl] | [Documentation][sanitize-htmlDOCUrl] | [![NPM version][sanitize-htmlNPMVersionImage]][sanitize-htmlNPMUrl] [select2][select2GHUrl] | [Documentation][select2DOCUrl] | [![NPM version][select2NPMVersionImage]][select2NPMUrl] [select2-bootstrap-css][select2-bootstrap-cssGHUrl] | [Documentation][select2-bootstrap-cssDOCUrl] | [![NPM version][select2-bootstrap-cssNPMVersionImage]][select2-bootstrap-cssNPMUrl] @@ -166,12 +165,6 @@ Outdated dependencies list can also be achieved with `$ npm --depth 0 outdated` [bootstrap-markdownNPMUrl]: https://www.npmjs.com/package/bootstrap-markdown [bootstrap-markdownNPMVersionImage]: https://img.shields.io/npm/v/bootstrap-markdown.svg?style=flat -[bundlerGHUrl]: https://github.com/bundler/bundler -[bundlerDOCUrl]: http://bundler.io/ -[bundlerGEMUrl]: http://rubygems.org/gems/bundler -[bundlerGEMVersionImage]: http://img.shields.io/gem/v/bundler.svg?style=flat -[bundlerGHReleasesUrl]: https://github.com/bundler/bundler/blob/master/CHANGELOG.md - [clipboardGHUrl]: https://github.com/zenorocha/clipboard.js [clipboardDOCUrl]: https://github.com/zenorocha/clipboard.js/blob/master/readme.md [clipboardNPMUrl]: https://www.npmjs.com/package/clipboard @@ -218,11 +211,6 @@ Outdated dependencies list can also be achieved with `$ npm --depth 0 outdated` [express-sessionNPMUrl]: https://www.npmjs.com/package/express-session [express-sessionNPMVersionImage]: https://img.shields.io/npm/v/express-session.svg?style=flat -[fakes3GHUrl]: https://github.com/jubos/fake-s3 -[fakes3DOCUrl]: http://www.rubydoc.info/gems/fakes3 -[fakes3GEMUrl]: http://rubygems.org/gems/fakes3 -[fakes3GEMVersionImage]: http://img.shields.io/gem/v/fakes3.svg?style=flat - [font-awesomeGHUrl]: https://github.com/FortAwesome/Font-Awesome [font-awesomeDOCUrl]: http://fontawesome.io/ [font-awesomeNPMUrl]: https://www.npmjs.com/package/font-awesome @@ -417,6 +405,11 @@ Outdated dependencies list can also be achieved with `$ npm --depth 0 outdated` [rfc2047NPMUrl]: https://www.npmjs.com/package/rfc2047 [rfc2047NPMVersionImage]: https://img.shields.io/npm/v/rfc2047.svg?style=flat +[s3rverGHUrl]: https://github.com/jamhall/s3rver +[s3rverDOCUrl]: https://github.com/jamhall/s3rver/blob/master/README.md +[s3rverNPMUrl]: https://www.npmjs.com/package/s3rver +[s3rverNPMVersionImage]: https://img.shields.io/npm/v/s3rver.svg?style=flat + [sanitize-htmlGHUrl]: https://github.com/punkave/sanitize-html [sanitize-htmlDOCUrl]: https://github.com/punkave/sanitize-html/blob/master/README.md [sanitize-htmlNPMUrl]: https://www.npmjs.com/package/sanitize-html diff --git a/controllers/scriptStorage.js b/controllers/scriptStorage.js index f33c753f2..854f699df 100644 --- a/controllers/scriptStorage.js +++ b/controllers/scriptStorage.js @@ -11,12 +11,14 @@ var statusError = require('../libs/debug').statusError; //--- Dependency inclusions var fs = require('fs'); var util = require('util'); +var http = require('http'); var _ = require('underscore'); var crypto = require('crypto'); var request = require('request'); var stream = require('stream'); var peg = require('pegjs'); var AWS = require('aws-sdk'); +var S3rver = require('s3rver'); var UglifyJS = require("uglify-es"); var rfc2047 = require('rfc2047'); var mediaType = require('media-type'); @@ -114,24 +116,64 @@ var parsers = (function () { exports.parsers = parsers; var bucketName = 'OpenUserJS.org'; +if (isDev) { + bucketName = bucketName.toLowerCase(); // NOTE: *S3rver* requirement +} var DEV_AWS_URL = null; +var devAWSURL = null; +var serverS3 = null; if (isPro) { AWS.config.update({ region: 'us-east-1' }); } else { - // You need to install (and ruby too): https://github.com/jubos/fake-s3 - // Then run the fakes3.sh script or: fakes3 -r fakeS3 -p 10001 DEV_AWS_URL = process.env.DEV_AWS_URL || 'http://localhost:10001'; AWS.config.update({ accessKeyId: 'fakeId', secretAccessKey: 'fakeKey', httpOptions: { proxy: DEV_AWS_URL, - agent: require('http').globalAgent // TODO: Move this up eventually + agent: http.globalAgent + } + }); + + devAWSURL = new URL(DEV_AWS_URL); + + serverS3 = new S3rver({ + hostname: devAWSURL.hostname, + port: devAWSURL.port, + directory: './S3rver' // WATCHPOINT: Technically this should be `..` Is probably upstream issue + }).run(function (aErr) { + if (aErr) { + console.error([ + colors.red('ERROR: S3rver not initialized'), + aErr + ].join('\n')); + return; } + console.log(colors.green('S3rver initialized')); + + var s3 = new AWS.S3(); + s3.createBucket({ Bucket: bucketName }, function (aErr) { + if (aErr) { + switch (aErr.statusCode) { + case 409: + console.log(colors.green('Default dev S3 bucket already exists')); + break; + default: + console.error([ + colors.red('Error creating default dev S3 bucket'), + aErr + ].join('\n')); + // fallthrough + } + return; + } else { + console.log(colors.green('Created default dev S3 bucket')); + } + }); }); } diff --git a/controllers/user.js b/controllers/user.js index 5292e01cc..59671edfe 100644 --- a/controllers/user.js +++ b/controllers/user.js @@ -1952,7 +1952,7 @@ function getExistingScript(aReq, aOptions, aAuthedUser, aCallback) { 'S3 GET (chunking indirect) ', aErr.code, 'for', installNameBase + (isLib ? '.js' : '.user.js'), - 'in the', bucketName, 'bucket\n' + + 'in the S3 bucket\n' + JSON.stringify(aErr, null, ' ') + '\n' + aErr.stack ); diff --git a/dev/postinstall.js b/dev/postinstall.js index a21bad071..d76d41b56 100644 --- a/dev/postinstall.js +++ b/dev/postinstall.js @@ -26,123 +26,6 @@ var tasks = [ aCallback(null, ['$ ' + cmd + '\n' + colors.gray(aStdout)]); }); }, - function (aStdouts, aCallback) { - var cmd = 'ruby -v'; - - exec(cmd, function (aErr, aStdout, aStderr) { - if (aErr) { - aCallback(aErr); - return; - } - - aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, aStdouts); - }); - }, - function (aStdouts, aCallback) { - var cmd = 'bundler -v'; - - exec(cmd, function (aErr, aStdout, aStderr) { - if (aErr) { - if (aErr.code === 127) { -// aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, false, aStdouts); - return; - } else { - aCallback(aErr); - return; - } - } - -// aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, true, aStdouts); - }); - }, - function (aSkip, aStdouts, aCallback) { - var cmd = 'sudo gem install bundler -v 1.16.4'; - - if (aSkip) { - aCallback(null, aStdouts); - return; - } - - console.log(colors.cyan('Installing *bundler* gem as global...')); - - exec(cmd, function (aErr, aStdout, aStderr) { - if (aErr) { - aCallback(aErr); - return; - } - -// aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, aStdouts); - }); - }, - function (aStdouts, aCallback) { - var cmd = 'bundler outdated'; - - exec(cmd, function (aErr, aStdout, aStderr) { - if (aErr) { - if (aErr.code === 7) { -// aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, false, aStdouts); - return; - } else { - aCallback(aErr); - return; - } - } - -// aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, true, aStdouts); - }); - }, - function (aSkip, aStdouts, aCallback) { - var cmd = 'bundler install'; - - if (aSkip) { - aCallback(null, aStdouts); - return; - } - - console.log(colors.cyan('Installing bundled gem(s) as global...')); - - exec(cmd, function (aErr, aStdout, aStderr) { - if (aErr) { - aCallback(aErr); - return; - } - -// aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, aStdouts); - }); - }, - function (aStdouts, aCallback) { - var cmd = 'gem list'; - - exec(cmd, function (aErr, aStdout, aStderr) { - if (aErr) { - aCallback(aErr); - return; - } - - aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, aStdouts); - }); - }, - function (aStdouts, aCallback) { - var cmd = 'gem outdated'; - - exec(cmd, function (aErr, aStdout, aStderr) { - if (aErr) { - aCallback(aErr); - return; - } - - aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout)); - aCallback(null, aStdouts); - }); - }, function (aStdouts, aCallback) { var cmd = 'npm -v'; diff --git a/package.json b/package.json index a64b5b433..50a03db75 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,9 @@ "underscore": "1.9.1", "useragent": "2.3.0" }, + "devDependencies": { + "s3rver": "2.2.6" + }, "optionalDependencies": { "kerberos": "1.0.0" },