CLDSRV-527

lib/api/api.js

@@ -52,6 +52,7 @@ const objectGetRetention = require('./objectGetRetention');
 const objectGetTagging = require('./objectGetTagging');
 const objectHead = require('./objectHead');
 const objectPut = require('./objectPut');
+const objectPost = require('./objectPost');
 const objectPutACL = require('./objectPutACL');
 const objectPutLegalHold = require('./objectPutLegalHold');
 const objectPutTagging = require('./objectPutTagging');
@@ -68,11 +69,128 @@ const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
 const parseCopySource = require('./apiUtils/object/parseCopySource');
 const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
 const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
+const { decryptToken } = require('./apiUtils/object/continueToken');
+const busboy = require('busboy');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { PassThrough } = require('stream');
+
 
 const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
 
 auth.setHandler(vault);
 
+function parseMultipartFormData(request, callback) {
+    let algoOK = false;
+    let credOK = false;
+    let dateOK = false;
+    let sigOK = false;
+    let policyOK = false;
+    request.formData = {};
+
+    const boundary = request.headers['content-type'].split('boundary=')[1];
+    const boundaryBuffer = Buffer.from(`--${boundary}`);
+    const newlineBuffer = Buffer.from('\r\n');
+
+    let buffer = Buffer.alloc(0);
+    let currentField = null;
+    let file = null;
+    let count = 0;
+
+    request.on('data', (chunk) => {
+        buffer = Buffer.concat([buffer, chunk]);
+
+        let boundaryIndex;
+        console.log('part count:', count++);
+
+        while ((boundaryIndex = buffer.indexOf(boundaryBuffer)) !== -1) {
+            let part = buffer.slice(0, boundaryIndex);
+            buffer = buffer.slice(boundaryIndex + boundaryBuffer.length);
+
+            if (part.length === 0) continue; // skip empty parts
+
+            let partToProcess = part;
+            if (part.indexOf(newlineBuffer) === 0) {
+                part = part.slice(newlineBuffer.length);
+            }
+
+            const headersEndIndex = partToProcess.indexOf(newlineBuffer + newlineBuffer);
+            const headers = partToProcess.slice(0, headersEndIndex).toString().split('\r\n');
+            let content = partToProcess.slice(headersEndIndex + newlineBuffer.length * 2);
+            if (content.slice(-2).equals(newlineBuffer)) {
+                content = content.slice(0, -2);
+            }
+
+            const contentDisposition = headers.find(header => header.startsWith('Content-Disposition'));
+            const contentTypeHeader = headers.find(header => header.startsWith('Content-Type'));
+            const mimetype = contentTypeHeader ? contentTypeHeader.split(': ')[1] : '';
+
+            if (contentDisposition) {
+                const nameMatch = contentDisposition.match(/name="([^"]+)"/);
+                const filenameMatch = contentDisposition.match(/filename="([^"]+)"/);
+
+                if (nameMatch) {
+                    const fieldname = nameMatch[1];
+                    if (filenameMatch) {
+                        // File field
+                        const filename = filenameMatch[1];
+
+                        // Remove the trailing CRLF from the content
+
+                        // 'Content-Disposition: form-data; name="file"; filename="test.txt"'
+
+                        //const mimetype = headers.find(header => header.startsWith('Content-Type')).split(': ')[1];
+
+                        file = new PassThrough();
+                        file.write(content);
+
+                        // Pipe the remaining data
+                        request.pipe(file);
+                        //request.pipe(fileStream);
+
+                        if (algoOK && credOK && dateOK && sigOK && policyOK) {
+                            callback(null, { file, fieldname, filename, boundaryBuffer, mimetype });
+                        }
+
+                        currentField = null;
+                    } else {
+                        // Regular field
+                        currentField = fieldname;
+                        request.formData[fieldname] = content.toString();
+
+                        if (fieldname === 'X-Amz-Algorithm') {
+                            algoOK = true;
+                        }
+                        if (fieldname === 'X-Amz-Credential') {
+                            credOK = true;
+                        }
+                        if (fieldname === 'X-Amz-Date') {
+                            dateOK = true;
+                        }
+                        if (fieldname === 'X-Amz-Signature') {
+                            sigOK = true;
+                        }
+                        if (fieldname === 'Policy') {
+                            const decrypted = decryptToken(request.formData.Policy);
+                            request.formData.decryptedPolicy = JSON.parse(decrypted);
+                            policyOK = true;
+                        }
+
+                        currentField = null;
+                    }
+                }
+            }
+        }
+    });
+
+    request.on('end', () => {
+        if (!algoOK || !credOK || !dateOK || !sigOK || !policyOK) {
+            callback(new Error('InvalidRequest'));
+        }
+    });
+}
+
 /* eslint-disable no-param-reassign */
 const api = {
     callApiMethod(apiMethod, request, response, log, callback) {
@@ -112,7 +230,7 @@ const api = {
 
         // no need to check auth on website or cors preflight requests
         if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
-        apiMethod === 'corsPreflight') {
+            apiMethod === 'corsPreflight') {
             request.actionImplicitDenies = false;
             return this[apiMethod](request, log, callback);
         }
@@ -158,7 +276,7 @@ const api = {
                 // second item checks s3:GetObject(Version)Tagging action
                 if (!authResults[1].isAllowed) {
                     log.trace('get tagging authorization denial ' +
-                    'from Vault');
+                        'from Vault');
                     returnTagCount = false;
                 }
             } else {
@@ -184,8 +302,108 @@ const api = {
             }
             return { returnTagCount, isImplicitDeny };
         }
+        let bb;
+        let fileEventData = null;
+
+        if (apiMethod === 'objectPost' && request.headers['content-type'].includes('multipart/form-data')) {
+            bb = busboy({ headers: request.headers });
+        }
 
         return async.waterfall([
+            next => {
+                if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
+                    return next(null);
+                }
+                if (apiMethod === 'objectPost' && request.headers['content-type'].includes('multipart/form-data')) {
+                    writeContinue(request, response);
+
+                    let algoOK = false;
+                    let credOK = false;
+                    let dateOK = false;
+                    let sigOK = false;
+                    let policyOK = false;
+                    request.formData = {};
+                    bb.on('field', (fieldname, val) => {
+                        request.formData[fieldname] = val;
+                        if (request.formData.Policy) {
+                            const decrypted = decryptToken(request.formData.Policy);
+                            request.formData.decryptedPolicy = JSON.parse(decrypted);
+                        }
+
+                        // TODO - put content type field for file in request
+                        if (fieldname === 'X-Amz-Algorithm') {
+                            algoOK = true;
+                        }
+                        if (fieldname === 'X-Amz-Credential') {
+                            credOK = true;
+                        }
+                        if (fieldname === 'X-Amz-Date') {
+                            dateOK = true;
+                        }
+                        if (fieldname === 'X-Amz-Signature') {
+                            sigOK = true;
+                        }
+                        if (fieldname === 'Policy') {
+                            policyOK = true;
+                        }
+                    });
+
+                    bb.on('file', (fieldname, file, filename, encoding, mimetype) => {
+                        fileEventData = { fieldname, file, filename, encoding, mimetype };
+                        if (algoOK && credOK && dateOK && sigOK && policyOK) {
+                            return next(null);
+                        }
+                    });
+
+                    bb.on('finish', () => {
+                        // if authorization field is not found, return error
+                        if (!algoOK || !credOK || !dateOK || !sigOK || !policyOK) {
+                            return next(errors.InvalidRequest);
+                        }
+                    });
+                    request.pipe(bb);
+
+                    // parseMultipartFormData(request, (err, data) => {
+                    //     if (err) {
+                    //         return next(err);
+                    //     }
+                    //     fileEventData = data;
+                    //     return next(null);
+                    // });
+                } else {
+                    // issue 100 Continue to the client
+                    writeContinue(request, response);
+                    const MAX_POST_LENGTH = request.method === 'POST' ?
+                        1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
+                    const post = [];
+                    let postLength = 0;
+                    request.on('data', chunk => {
+                        postLength += chunk.length;
+                        // Sanity check on post length
+                        if (postLength <= MAX_POST_LENGTH) {
+                            post.push(chunk);
+                        }
+                    });
+
+                    request.on('error', err => {
+                        log.trace('error receiving request', {
+                            error: err,
+                        });
+                        return next(errors.InternalError);
+                    });
+
+                    request.on('end', () => {
+                        if (postLength > MAX_POST_LENGTH) {
+                            log.error('body length is too long for request type',
+                                { postLength });
+                            return next(errors.InvalidRequest);
+                        }
+                        request.post = Buffer.concat(post, postLength).toString();
+                        return next(null);
+                    });
+                }
+                return undefined;
+            },
             next => auth.server.doAuth(
                 request, log, (err, userInfo, authorizationResults, streamingV4Params) => {
                     if (err) {
@@ -200,41 +418,7 @@ const api = {
                     authNames.userName = userInfo.getIAMdisplayName();
                 }
                 log.addDefaultFields(authNames);
-                if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
-                    return next(null, userInfo, authorizationResults, streamingV4Params);
-                }
-                // issue 100 Continue to the client
-                writeContinue(request, response);
-                const MAX_POST_LENGTH = request.method === 'POST' ?
-                      1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
-                const post = [];
-                let postLength = 0;
-                request.on('data', chunk => {
-                    postLength += chunk.length;
-                    // Sanity check on post length
-                    if (postLength <= MAX_POST_LENGTH) {
-                        post.push(chunk);
-                    }
-                });
-
-                request.on('error', err => {
-                    log.trace('error receiving request', {
-                        error: err,
-                    });
-                    return next(errors.InternalError);
-                });
-
-                request.on('end', () => {
-                    if (postLength > MAX_POST_LENGTH) {
-                        log.error('body length is too long for request type',
-                                  { postLength });
-                        return next(errors.InvalidRequest);
-                    }
-                    // Convert array of post buffers into one string
-                    request.post = Buffer.concat(post, postLength).toString();
-                    return next(null, userInfo, authorizationResults, streamingV4Params);
-                });
-                return undefined;
+                return next(null, userInfo, authorizationResults, streamingV4Params);
             },
             // Tag condition keys require information from CloudServer for evaluation
             (userInfo, authorizationResults, streamingV4Params, next) => tagConditionKeyAuth(
@@ -244,6 +428,10 @@ const api = {
                 apiMethod,
                 log,
                 (err, authResultsWithTags) => {
+                    // TODO CLDSRV-527 remove ignore for POST object here
+                    if (apiMethod === 'objectPost') {
+                        return next(null, userInfo, authorizationResults, streamingV4Params);
+                    }
                     if (err) {
                         log.trace('tag authentication error', { error: err });
                         return next(err);
@@ -271,6 +459,13 @@ const api = {
                     return acc;
                 }, {});
             }
+            if (apiMethod === 'objectPost' && fileEventData) {
+                request._response = response;
+                request.file = fileEventData.file;
+                request.fileEventData = fileEventData;
+                return this[apiMethod](userInfo, request, streamingV4Params,
+                    log, callback, authorizationResults);
+            }
             if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
                 request._response = response;
                 return this[apiMethod](userInfo, request, streamingV4Params,
@@ -337,6 +532,7 @@ const api = {
     objectCopy,
     objectHead,
     objectPut,
+    objectPost,
     objectPutACL,
     objectPutLegalHold,
     objectPutTagging,

lib/api/apiUtils/object/createAndStoreObject.js

@@ -210,6 +210,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
                 metadataStoreParams.contentMD5 = constants.emptyFileMd5;
                 return next(null, null, null);
             }
+            if (request.apiMethod === 'objectPost') {
+                return dataStore(objectKeyContext, cipherBundle, request.file, size,
+                    streamingV4Params, backendInfo, log, next);
+            }
             return dataStore(objectKeyContext, cipherBundle, request, size,
                     streamingV4Params, backendInfo, log, next);
         },

lib/api/apiUtils/object/prepareStream.js

@@ -13,7 +13,7 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform');
  * the type of request requires them
  */
 function prepareStream(stream, streamingV4Params, log, errCb) {
-    if (stream.headers['x-amz-content-sha256'] ===
+    if (stream.headers && stream.headers['x-amz-content-sha256'] ===
         'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') {
         if (typeof streamingV4Params !== 'object') {
             // this might happen if the user provided a valid V2