Add plimit and increase cache

.github/workflows/test.yml

@@ -14,9 +14,9 @@ jobs:
         os: [ubuntu-22.04]
         include:
           - node: 16
-            os: macos-12
+            os: macos-14
           - node: 20
-            os: macos-12
+            os: macos-14
     runs-on: ${{ matrix.os }}
     steps:
       - name: Install node-gyp deps

binding.gyp

@@ -3,47 +3,69 @@
         {
             "target_name": "jq-node-bindings",
             "sources": [
-                "src/binding.cc",
+                "src/binding.cc"
             ],
             "include_dirs": [
-                "<!(node -e \"require('nan')\")",
+                "<!(node -p \"require('node-addon-api').include_dir\")",
                 "<(module_root_dir)/",
-                "deps/jq/src"
+                "deps/jq/src",
+                "../node-addon-api/"
             ],
-            'conditions': [
+            "defines": [
+                "NAPI_VERSION=8",
+                'NAPI_DISABLE_CPP_EXCEPTIONS'
+            ],
+            "conditions": [
                 [
-                    'OS=="linux"',
+                    "OS=='linux'",
                     {
                         "libraries": [
                             "-Wl,-rpath='$$ORIGIN/../deps'",
-                            "../build/deps/libjq.so.1",
+                            "../build/deps/libjq.so.1"
                         ],
-                        'cflags_cc': [
-                            '-std=c++17'
+                        "cflags_cc": [
+                            "-std=c++17"
                         ],
-                        'cflags_cc!': [
-                            '-fno-rtti -fno-exceptions'
+                        "cflags_cc!": [
+                            "-fno-rtti -fno-exceptions"
                         ]
                     }
                 ],
                 [
-                    'OS=="mac"',
+                    "OS=='mac'",
                     {
                         "libraries": [
-                            "../build/deps/libjq.dylib",
-                            # "../build/deps/libonig.4.dylib",
-                            # "../build/deps/libonig.dylib",
+                            "../build/deps/libjq.dylib"
                         ],
-                        'xcode_settings': {
-                            'MACOSX_DEPLOYMENT_TARGET': '12.0.1',
-                            'GCC_ENABLE_CPP_RTTI': 'YES',
-                            'GCC_ENABLE_CPP_EXCEPTIONS': 'YES'
+                        "xcode_settings": {
+                            "MACOSX_DEPLOYMENT_TARGET": "12.0.1",
+                            "GCC_ENABLE_CPP_RTTI": "YES",
+                            "GCC_ENABLE_CPP_EXCEPTIONS": "YES"
                         },
-                        'OTHER_CPLUSPLUSFLAGS': [
-                            '-std=c++17'
+                        "OTHER_CPLUSPLUSFLAGS": [
+                            "-std=c++17"
                         ],
                         "include_dirs": [
                             "deps/jq/src"
+                        ],      "cflags": ["-fsanitize=address", "-fno-omit-frame-pointer"],
+      "ldflags": ["-fsanitize=address"],
+                    }
+                ],
+                [
+                    "OS=='win'",
+                    {
+                        "msvs_settings": {
+                            "VCCLCompilerTool": {
+                                "AdditionalOptions": ["/std:c++17"],
+                                "ExceptionHandling": 1,
+                                "RuntimeTypeInfo": "true"
+                            }
+                        },
+                        "libraries": [
+                            "deps\\jq\\build\\Release\\libjq.lib"
+                        ],
+                        "include_dirs": [
+                            "deps\\jq\\src"
                         ]
                     }
                 ]

configure

@@ -20,7 +20,7 @@ make -j8
 cp modules/oniguruma/src/.libs/libonig.a ${scriptdir}/build/deps/libonig.a
 cp modules/oniguruma/src/.libs/libonig.la ${scriptdir}/build/deps/libonig.la
 cp modules/oniguruma/src/.libs/libonig.lai ${scriptdir}/build/deps/libonig.lai
-cp modules/oniguruma/src/.libs/libonig.4.dylib ${scriptdir}/build/deps/libonig.4.dylib
+cp modules/oniguruma/src/.libs/libonig.5.dylib ${scriptdir}/build/deps/libonig.5.dylib
 cp modules/oniguruma/src/.libs/libonig.dylib ${scriptdir}/build/deps/libonig.dylib
 
 make install-libLTLIBRARIES install-includeHEADERS

index.d.ts

@@ -8,6 +8,11 @@ declare module '@port-labs/jq-node-bindings' {
   }
 
   export function exec(json: object, input: string, options?: ExecOptions): object | Array<any> | string | number | boolean | null;
+  export function execAsync(json: object, input: string, options?: ExecOptions): Promise<object | Array<any> | string | number | boolean | null>;
+
+  export function setLimit(limit: number): void;
+  export function setCacheSize(size: number): void;
 
   export function renderRecursively(json: object, input: object | Array<any> | string | number | boolean | null, execOptions?: ExecOptions): object | Array<any> | string | number | boolean | null;
+  export function renderRecursivelyAsync(json: object, input: object | Array<any> | string | number | boolean | null, execOptions?: ExecOptions): Promise<object | Array<any> | string | number | boolean | null>;
 }

lib/index.js

@@ -1,10 +1,13 @@
 const jq = require('./jq');
 const template = require('./template');
+const templateAsync = require('./templateAsync');
 
 
 module.exports = {
   exec: jq.exec,
+  execAsync: jq.execAsync,
   renderRecursively: template.renderRecursively,
+  renderRecursivelyAsync: templateAsync.renderRecursivelyAsync,
   JqExecError: jq.JqExecError,
   JqExecCompileError: jq.JqExecCompileError,
 };

lib/jq.js

@@ -1,4 +1,24 @@
 const nativeJq = require('bindings')('jq-node-bindings')
+let pLimit = null
+let currentLimit = 2
+
+const initPLimit = async () => {
+  if (!pLimit) {
+    const module = await import('p-limit')
+    pLimit = module.default(currentLimit)
+  }
+  return pLimit
+}
+
+const setLimit = (limit) => {
+  if (typeof limit !== 'number' || limit < 1) {
+    throw new Error('Limit must be a positive number')
+  }
+  currentLimit = limit
+  pLimit = null
+}
+
+nativeJq.setCacheSize(2000)
 
 const formatFilter = (filter, {enableEnv = false} = {}) => {
   // Escape single quotes only if they are opening or closing a string
@@ -16,7 +36,7 @@ class JqExecCompileError extends JqExecError {
 
 const exec = (object, filter, {enableEnv = false, throwOnError = false} = {}) => {
   try {
-    const data = nativeJq.exec(JSON.stringify(object), formatFilter(filter, {enableEnv}))
+    const data = nativeJq.execSync(JSON.stringify(object), formatFilter(filter, {enableEnv}))
 
     return data?.value;
   } catch (err) {
@@ -27,8 +47,23 @@ const exec = (object, filter, {enableEnv = false, throwOnError = false} = {}) =>
   }
 }
 
+const execAsync = async (object, filter, {enableEnv = false, throwOnError = false} = {}) => {
+  const limit = await initPLimit()
+  try {
+    const data = await limit(() => nativeJq.execAsync(JSON.stringify(object), formatFilter(filter, {enableEnv})))
+    return data?.value;
+  } catch (err) {
+    if (throwOnError) {
+      throw new (err?.message?.startsWith('jq: compile error') ? JqExecCompileError : JqExecError)(err.message);
+    }
+    return null
+  }
+}
 module.exports = {
   exec,
+  execAsync,
+  setLimit,
+  setCacheSize:nativeJq.setCacheSize,
   JqExecError,
   JqExecCompileError
 };

lib/templateAsync.js

@@ -0,0 +1,136 @@
+const jq = require('./jq');
+
+const findInsideDoubleBracesIndices = (input) => {
+  let wrappingQuote = null;
+  let insideDoubleBracesStart = null;
+  const indices = [];
+
+  for (let i = 0; i < input.length; i += 1) {
+    const char = input[i];
+
+    if (insideDoubleBracesStart && char === '\\') {
+      // If next character is escaped, skip it
+      i += 1;
+    }
+    if (insideDoubleBracesStart && (char === '"' || char === "'")) {
+      // If inside double braces and inside quotes, ignore braces
+      if (!wrappingQuote) {
+        wrappingQuote = char;
+      } else if (wrappingQuote === char) {
+        wrappingQuote = null;
+      }
+    } else if (!wrappingQuote && char === '{' && i > 0 && input[i - 1] === '{') {
+      // if opening double braces that not wrapped with quotes
+      if (insideDoubleBracesStart) {
+        throw new Error(`Found double braces in index ${i - 1} inside other one in index ${insideDoubleBracesStart - '{{'.length}`);
+      }
+      insideDoubleBracesStart = i + 1;
+      if (input[i + 1] === '{') {
+        // To overcome three "{" in a row considered as two different opening double braces
+        i += 1;
+      }
+    } else if (!wrappingQuote && char === '}' && i > 0 && input[i - 1] === '}') {
+      // if closing double braces that not wrapped with quotes
+      if (insideDoubleBracesStart) {
+        indices.push({start: insideDoubleBracesStart, end: i - 1});
+        insideDoubleBracesStart = null;
+        if (input[i + 1] === '}') {
+          // To overcome three "}" in a row considered as two different closing double braces
+          i += 1;
+        }
+      } else {
+        throw new Error(`Found closing double braces in index ${i - 1} without opening double braces`);
+      }
+    }
+  }
+
+  if (insideDoubleBracesStart) {
+    throw new Error(`Found opening double braces in index ${insideDoubleBracesStart - '{{'.length} without closing double braces`);
+  }
+
+  return indices;
+}
+
+const renderAsync =async (inputJson, template, execOptions = {}) => {
+  if (typeof template !== 'string') {
+    return null;
+  }
+  const indices = findInsideDoubleBracesIndices(template);
+  if (!indices.length) {
+    // If no jq templates in string, return it
+    return template;
+  }
+
+  const firstIndex = indices[0];
+  if (indices.length === 1 && template.trim().startsWith('{{') && template.trim().endsWith('}}')) {
+    // If entire string is a template, evaluate and return the result with the original type
+    return await jq.execAsync(inputJson, template.slice(firstIndex.start, firstIndex.end), execOptions);
+  }
+
+  let result = template.slice(0, firstIndex.start - '{{'.length); // Initiate result with string until first template start index
+  for (let i = 0; i < indices.length; i++) {
+    const index = indices[i];
+
+//   }
+//   indices.forEach((index, i) => {
+    const jqResult = await jq.execAsync(inputJson, template.slice(index.start, index.end), execOptions);
+    result +=
+      // Add to the result the stringified evaluated jq of the current template
+      (typeof jqResult === 'string' ? jqResult : JSON.stringify(jqResult)) +
+      // Add to the result from template end index. if last template index - until the end of string, else until next start index
+      template.slice(
+        index.end + '}}'.length,
+        i + 1 === indices.length ? template.length : indices[i + 1].start - '{{'.length,
+      );
+//   });
+    }
+
+  return result;
+}
+
+const renderRecursivelyAsync = async(inputJson, template, execOptions = {}) => {
+  if (typeof template === 'string') {
+    return await renderAsync(inputJson, template, execOptions);
+  }
+  if (Array.isArray(template)) {
+    return await Promise.all(template.map((value) => renderRecursivelyAsync(inputJson, value, execOptions)));
+  }
+  if (typeof template === 'object' && template !== null) {
+
+
+
+      const t =Object.entries(template).map(async([key, value]) => {
+        const SPREAD_KEYWORD = "spreadValue";
+        const keywordMatcher = `^\\{\\{\\s*${SPREAD_KEYWORD}\\(\\s*\\)\\s*\\}\\}$`;  // matches {{ <Keyword>() }} with white spaces where you'd expect them
+
+        if (key.trim().match(keywordMatcher)) {
+          const evaluatedValue = await renderRecursivelyAsync(inputJson, value, execOptions);
+          if (typeof evaluatedValue !== "object") {
+            throw new Error(
+              `Evaluated value should be an object if the key is ${key}. Original value: ${value}, evaluated to: ${JSON.stringify(evaluatedValue)}`
+            );
+          }
+          return Object.entries(evaluatedValue);
+        }
+
+        const evaluatedKey = await renderRecursivelyAsync(inputJson, key, execOptions);
+        if (!['undefined', 'string'].includes(typeof evaluatedKey) && evaluatedKey !== null) {
+          throw new Error(
+            `Evaluated object key should be undefined, null or string. Original key: ${key}, evaluated to: ${JSON.stringify(evaluatedKey)}`,
+          );
+        }
+        return evaluatedKey ? [[evaluatedKey, await renderRecursivelyAsync(inputJson, value, execOptions)]] : [];
+      });
+
+
+    return Object.fromEntries((await Promise.all(t)).flat());
+
+
+  }
+
+  return template;
+}
+
+module.exports = {
+  renderRecursivelyAsync
+};