Merge branch 'v1.2'

Author: proletesseract

.eslintrc

@@ -31,6 +31,6 @@
     "prefer-template": 0,
     "no-restricted-syntax": 0,
     "strict": 0,
-    "no-underscore-dangle": ["error", {"allow": ["__set__"]}]
+    "no-underscore-dangle": ["warn", {"allow": ["__set__", "_*"]}]
   }
 }

.gitignore

@@ -7,3 +7,5 @@ log.txt
 npm-debug.log
 config/default.json
 .idea/
+
+\.DS_Store

dist/navtech.js

@@ -245,6 +245,7 @@ const apiInit = () => {
       }))
       return
     }
+    // Logger.writeLog('APP_TEST_001', 'success get-addresses', { data })
     NavtechApi.runtime.res.send(JSON.stringify({
       status: 200,
       type: 'SUCCESS',
@@ -324,6 +325,18 @@ const apiInit = () => {
       return
     }
 
+    if (globalSettings.serverType === 'INCOMING' && IncomingServer.paused
+       || globalSettings.serverType === 'OUTGOING' && OutgoingServer.paused) {
+      Logger.writeLog('APP_026A', 'this server is paused for manual recovery', { body: req.body })
+      NavtechApi.runtime.res.send(JSON.stringify({
+        status: 200,
+        type: 'FAIL',
+        code: 'APP_026A',
+        message: 'server is not accepting transactions',
+      }))
+      return
+    }
+
     NavtechApi.runtime.numAddresses = parseInt(NavtechApi.runtime.req.body.num_addresses, 10)
 
     if (globalSettings.serverType === 'INCOMING') {
@@ -452,7 +465,7 @@ const apiInit = () => {
           highestConf = pending.confirmations
         }
       }
-      if (highestConf > 60) {
+      if (highestConf > privateSettings.maxQueue) {
         Logger.writeLog('APP_030B', 'the queue is too long', { highestConf }, true)
         NavtechApi.runtime.res.send(JSON.stringify({
           status: 200,
@@ -571,13 +584,20 @@ const apiInit = () => {
 
     returnData.nav_addresses = NavtechApi.runtime.navAddresses
 
+    // Logger.writeLog('APP_TEST_002', 'success check-node', {
+    //   returnData,
+    //   request: NavtechApi.runtime.req.body,
+    // })
+
     NavtechApi.runtime.res.send(JSON.stringify({
       status: 200,
       type: 'SUCCESS',
       data: returnData,
     }))
   }
 
+  // @TODO check if server paused before returning as valid incoming server
+
   // -------------- CHECK IF SERVER IS PROCESSING ------------------------------------------------------------------------------------------
 
   app.get('/api/status', (req, res) => {

dist/vendor.js

@@ -2,17 +2,18 @@
 
 const Client = require('bitcoin-core')
 const config = require('config')
+const lodash = require('lodash')
 
-const EncryptionKeys = require('./lib/EncryptionKeys.js')
-const Logger = require('./lib/Logger.js')
-const PreFlight = require('./lib/PreFlight.js')
-const RefillOutgoing = require('./lib/RefillOutgoing.js')
-const SelectOutgoing = require('./lib/SelectOutgoing.js')
-const ReturnAllToSenders = require('./lib/ReturnAllToSenders.js')
-const PrepareIncoming = require('./lib/PrepareIncoming.js')
-const RetrieveSubchainAddresses = require('./lib/RetrieveSubchainAddresses.js')
-const ProcessIncoming = require('./lib/ProcessIncoming.js')
-const SpendToHolding = require('./lib/SpendToHolding.js')
+let EncryptionKeys = require('./lib/EncryptionKeys.js') //eslint-disable-line
+let Logger = require('./lib/Logger.js') //eslint-disable-line
+let PreFlight = require('./lib/PreFlight.js') //eslint-disable-line
+let RefillOutgoing = require('./lib/RefillOutgoing.js') //eslint-disable-line
+let SelectOutgoing = require('./lib/SelectOutgoing.js') //eslint-disable-line
+let ReturnAllToSenders = require('./lib/ReturnAllToSenders.js') //eslint-disable-line
+let PrepareIncoming = require('./lib/PrepareIncoming.js') //eslint-disable-line
+let RetrieveSubchainAddresses = require('./lib/RetrieveSubchainAddresses.js') //eslint-disable-line
+let ProcessIncoming = require('./lib/ProcessIncoming.js') //eslint-disable-line
+let SpendToHolding = require('./lib/SpendToHolding.js') //eslint-disable-line
 
 const settings = config.get('INCOMING')
 
@@ -41,10 +42,11 @@ IncomingServer.init = () => {
 
   Logger.writeLog('INC_000', 'server starting')
   EncryptionKeys.findKeysToRemove({ type: 'private' }, IncomingServer.startProcessing)
-  setInterval(() => {
+  IncomingServer.cron = setInterval(() => {
     if (IncomingServer.paused === false) {
       EncryptionKeys.findKeysToRemove({ type: 'private' }, IncomingServer.startProcessing)
     } else {
+      clearInterval(IncomingServer.cron)
       Logger.writeLog('INC_001', 'processing paused', { paused: IncomingServer.paused })
     }
   }, settings.scriptInterval)
@@ -80,6 +82,7 @@ IncomingServer.holdingProcessed = (success, data) => {
   if (!success) {
     Logger.writeLog('INC_004', 'failed to process the holding account', { success, data }, true)
     IncomingServer.processing = false
+    IncomingServer.paused = true
     return
   }
   SelectOutgoing.run({
@@ -96,6 +99,9 @@ IncomingServer.outgoingSelected = (success, data) => {
   }
 
   if (data.returnAllToSenders) {
+    if (data.pause) {
+      IncomingServer.paused = true
+    }
     ReturnAllToSenders.run({
       navClient: IncomingServer.navClient,
     }, IncomingServer.allPendingReturned)
@@ -111,14 +117,15 @@ IncomingServer.outgoingSelected = (success, data) => {
     navClient: IncomingServer.navClient,
     outgoingNavBalance: data.outgoingNavBalance,
     subBalance: IncomingServer.runtime.subBalance,
+    settings,
   }, IncomingServer.currentBatchPrepared)
 }
 
 IncomingServer.allPendingReturned = (success, data) => {
-  console.log('STATUS: IncomingServer.allPendingReturned', success, data)
   if (!success) {
     Logger.writeLog('INC_006', 'failed to return all pending to sender', { success, data }, true)
     IncomingServer.processing = false
+    IncomingServer.paused = true
     return
   }
   Logger.writeLog('INC_007', 'returned all pending to sender', { success, data }, true)
@@ -127,15 +134,55 @@ IncomingServer.allPendingReturned = (success, data) => {
 }
 
 IncomingServer.currentBatchPrepared = (success, data) => {
-  if (!success || !data || !data.currentBatch) {
+  if (!success || !data || ((!data.currentBatch || !data.currentFlattened || !data.numFlattened) && !data.pendingToReturn)) {
+    Logger.writeLog('INC_011D', 'prepareIncoming returned bad data', { success, data })
     IncomingServer.processing = false
     return
   }
+
   IncomingServer.runtime.currentBatch = data.currentBatch
+  IncomingServer.runtime.currentFlattened = data.currentFlattened
+  IncomingServer.runtime.numFlattened = data.numFlattened
+  IncomingServer.runtime.pendingToReturn = data.pendingToReturn
+
+  if (IncomingServer.runtime.pendingToReturn && IncomingServer.runtime.pendingToReturn.length > 0) {
+    Logger.writeLog('INC_011', 'failed to process some transactions', { success, data }, true)
+    ReturnAllToSenders.fromList({
+      navClient: IncomingServer.navClient,
+      transactionsToReturn: IncomingServer.runtime.pendingToReturn,
+    }, IncomingServer.pendingFailedReturned)
+    return
+  }
+
+  if (!IncomingServer.runtime.currentBatch || lodash.size(IncomingServer.runtime.currentBatch) === 0) {
+    Logger.writeLog('INC_011B', 'no currentBatch to process', { currentBatch: IncomingServer.runtime.currentBatch })
+    IncomingServer.processing = false
+    return
+  }
+  RetrieveSubchainAddresses.run({
+    subClient: IncomingServer.subClient,
+    chosenOutgoing: IncomingServer.runtime.chosenOutgoing,
+    numAddresses: IncomingServer.runtime.numFlattened,
+  }, IncomingServer.retrievedSubchainAddresses)
+}
+
+IncomingServer.pendingFailedReturned = (success, data) => {
+  if (!success) {
+    Logger.writeLog('INC_011A', 'failed to return failed pending to sender', { success, data }, true)
+    IncomingServer.paused = true
+    ReturnAllToSenders.run({
+      navClient: IncomingServer.navClient,
+    }, IncomingServer.allPendingReturned)
+  }
+  if (!IncomingServer.runtime.currentBatch || lodash.size(IncomingServer.runtime.currentBatch) === 0) {
+    Logger.writeLog('INC_011C', 'no currentBatch to process', { currentBatch: IncomingServer.runtime.currentBatch })
+    IncomingServer.processing = false
+    return
+  }
   RetrieveSubchainAddresses.run({
     subClient: IncomingServer.subClient,
     chosenOutgoing: IncomingServer.runtime.chosenOutgoing,
-    currentBatch: data.currentBatch,
+    numAddresses: IncomingServer.runtime.numFlattened,
   }, IncomingServer.retrievedSubchainAddresses)
 }
 
@@ -147,8 +194,10 @@ IncomingServer.retrievedSubchainAddresses = (success, data) => {
     }, IncomingServer.allPendingReturned)
     return
   }
+  // @TODO compile the correct transactions to return
   ProcessIncoming.run({
     currentBatch: IncomingServer.runtime.currentBatch,
+    currentFlattened: IncomingServer.runtime.currentFlattened,
     outgoingPubKey: IncomingServer.runtime.outgoingPubKey,
     subClient: IncomingServer.subClient,
     navClient: IncomingServer.navClient,
@@ -159,21 +208,38 @@ IncomingServer.retrievedSubchainAddresses = (success, data) => {
 
 IncomingServer.transactionsProcessed = (success, data) => {
   if (!success || !data) {
+    if (data && data.partialFailure) {
+      Logger.writeLog('INC_010A', 'failed part way through processing subchain transactions', { success, data }, true)
+      IncomingServer.paused = true
+      IncomingServer.processing = false
+      return
+    }
     Logger.writeLog('INC_010', 'failed to process transactions', { success, data }, true)
+    IncomingServer.paused = true
     ReturnAllToSenders.run({
       navClient: IncomingServer.navClient,
     }, IncomingServer.allPendingReturned)
     return
   }
 
-  IncomingServer.runtime.successfulSubTransactions = data.successfulSubTransactions
-  IncomingServer.runtime.transactionsToReturn = data.transactionsToReturn
+  IncomingServer.runtime.successfulTxGroups = data.successfulTxGroups
+  IncomingServer.runtime.txGroupsToReturn = data.txGroupsToReturn
+  IncomingServer.runtime.transactionsToReturn = []
 
-  if (IncomingServer.runtime.transactionsToReturn && IncomingServer.runtime.transactionsToReturn.length > 0) {
+  if (IncomingServer.runtime.txGroupsToReturn && IncomingServer.runtime.txGroupsToReturn.length > 0) {
     Logger.writeLog('INC_011', 'failed to process some transactions', { success, data }, true)
+
+    // extract the relevant transactions to return from the txGroupsToReturn
+    for (let i = 0; i < IncomingServer.runtime.txGroupsToReturn.length; i++) {
+      const txGroup = IncomingServer.runtime.txGroupsToReturn[i]
+      for (let j = 0; j < txGroup.transactions.length; j++) {
+        IncomingServer.runtime.transactionsToReturn.push(txGroup.transactions[j])
+      }
+    }
+
     ReturnAllToSenders.fromList({
       navClient: IncomingServer.navClient,
-      transactionsToReturn: data.transactionsToReturn,
+      transactionsToReturn: IncomingServer.runtime.transactionsToReturn,
     }, IncomingServer.failedTransactionsReturned)
     return
   }
@@ -182,8 +248,18 @@ IncomingServer.transactionsProcessed = (success, data) => {
 
 IncomingServer.failedTransactionsReturned = (success, data) => {
   if (!success) {
+    IncomingServer.paused = true
     Logger.writeLog('INC_012', 'failed to return failed transactions to sender', { success, data }, true)
   }
+  IncomingServer.runtime.successfulSubTransactions = []
+  // extract the relevant transactions to return from the txGroupsToReturn
+  for (let i = 0; i < IncomingServer.runtime.successfulTxGroups.length; i++) {
+    const txGroup = IncomingServer.runtime.successfulTxGroups[i]
+    lodash.forEach(txGroup.transactions, (transaction) => {
+      IncomingServer.runtime.successfulSubTransactions.push(transaction)
+    })
+  }
+
   SpendToHolding.run({
     successfulSubTransactions: IncomingServer.runtime.successfulSubTransactions,
     holdingEncrypted: IncomingServer.runtime.holdingEncrypted,

src/app.js

@@ -63,7 +63,8 @@ EncryptedData.decryptData = (options, callback) => {
           const key = ursa.createPrivateKey(fs.readFileSync(keyFile))
           const msg = key.decrypt(options.encryptedData, 'base64', 'utf8', ursa.RSA_PKCS1_PADDING)
           successfulDecryption = true
-          decrypted = (globalSettings.serverType === 'INCOMING') ? msg : JSON.parse(msg)
+          // Logger.writeLog('ECD_TEST', 'decrypted', { msg })
+          decrypted = JSON.parse(msg)
         } catch (err2) {
           // do nothing
         }

src/incoming.js

@@ -0,0 +1,74 @@
+const lodash = require('lodash')
+
+let Logger = require('./Logger.js') //eslint-disable-line
+
+const FlattenTransactions = {}
+
+FlattenTransactions.incoming = (options, callback) => {
+  const required = ['amountToFlatten', 'anonFeePercent']
+  if (lodash.intersection(Object.keys(options), required).length !== required.length) {
+    Logger.writeLog('FLT_001', 'invalid options', { options, required })
+    callback(false, { message: 'invalid options provided to FlattenTransactions.incoming' })
+    return
+  }
+  const unsafeAmount = options.amountToFlatten / (1 + (options.anonFeePercent / 100))
+  FlattenTransactions.runtime = {
+    callback,
+    amountToFlatten: FlattenTransactions.satoshiParser(unsafeAmount),
+  }
+  FlattenTransactions.flattenIncoming()
+}
+
+FlattenTransactions.flattenIncoming = () => {
+  const totalInt = Math.floor(FlattenTransactions.runtime.amountToFlatten)
+  const totalIntString = totalInt.toString()
+  const decimal = FlattenTransactions.runtime.amountToFlatten - totalInt
+  const safeDecimal = FlattenTransactions.satoshiParser(decimal)
+
+  let flattened = []
+
+  for (let i = 0; i < totalIntString.length; i++) {
+    const factor = 1 * Math.pow(10, totalIntString.length - (i + 1))
+    const numFactors = parseInt(totalIntString[i], 10)
+    for (let j = 0; j < numFactors; j++) {
+      if (safeDecimal > 0 && lodash.sum(flattened) === totalInt - factor) {
+        flattened.push(FlattenTransactions.satoshiParser(parseInt(factor, 10) + safeDecimal))
+      } else {
+        flattened.push(parseInt(factor, 10))
+      }
+    }
+  }
+
+  if (flattened.length === 1) {
+    flattened = []
+    for (let k = 0; k < 10; k++) {
+      if (safeDecimal > 0 && k === 9) {
+        flattened.push(FlattenTransactions.satoshiParser((totalInt / 10) + safeDecimal))
+      } else {
+        flattened.push(totalInt / 10)
+      }
+    }
+  }
+
+  const reduced = flattened.reduce((acc, x) => x + acc, 0)
+  const safeReduced = FlattenTransactions.satoshiParser(reduced)
+
+  if (safeReduced !== FlattenTransactions.runtime.amountToFlatten) {
+    Logger.writeLog('FLT_002', 'unable to correctly flatten amount', { runtime: FlattenTransactions.runtime, flattened })
+    FlattenTransactions.runtime.callback(false, {
+      flattened,
+    })
+    return
+  }
+
+  FlattenTransactions.runtime.callback(true, {
+    flattened,
+  })
+}
+
+FlattenTransactions.satoshiParser = (unsafe) => {
+  const satoshiFactor = 100000000
+  return Math.round(unsafe * satoshiFactor) / satoshiFactor
+}
+
+module.exports = FlattenTransactions