Add topic caching to dependent examples

Author: milindl

examples/kafkajs/admin/dependent_admin.js renamed to examples/kafkajs/admin/dependent-admin.js

@@ -3,9 +3,9 @@ const { Kafka } = require('@confluentinc/kafka-javascript').KafkaJS;
 
 async function adminFromConsumer() {
     const kafka = new Kafka({
-      kafkaJS: {
-        brokers: ['localhost:9092'],
-      }
+        kafkaJS: {
+            brokers: ['localhost:9092'],
+        }
     });
 
     const consumer = kafka.consumer({
@@ -49,18 +49,17 @@ async function adminFromConsumer() {
 
 async function adminFromProducer() {
     const kafka = new Kafka({
-      kafkaJS: {
-        brokers: ['localhost:9092'],
-      }
+        kafkaJS: {
+            brokers: ['localhost:9092'],
+        }
     });
 
-    const producer = kafka.producer({});
+    const producer = kafka.producer({
+        'metadata.max.age.ms': 900000, /* This is set to the default value. */
+    });
 
     await producer.connect();
 
-    // The producer can be used as normal
-    await producer.send({ topic: 'test-topic', messages: [{ value: 'Hello!' }] });
-
     // And the same producer can create an admin client - the producer must have successfully
     // been connected before the admin client can be created.
     const admin = producer.dependentAdmin();
@@ -70,6 +69,23 @@ async function adminFromProducer() {
     const listTopicsResult = await admin.listTopics();
     console.log(listTopicsResult);
 
+    // A common use case for the dependent admin client is to make sure the topic
+    // is cached before producing to it. This avoids delay in sending the first
+    // message to any topic. Using the admin client linked to the producer allows
+    // us to do this, by calling `fetchTopicMetadata` before we produce.
+    // Here, we cache all possible topics, but it's advisable to only cache the
+    // topics you are going to produce to (if you know it in advance),
+    // and avoid calling listTopics().
+    // Once a topic is cached, it will stay cached for `metadata.max.age.ms`,
+    // which is 15 minutes by default, after which it will be removed if
+    // it has not been produced to.
+    await admin.fetchTopicMetadata({ topics: listTopicsResult }).catch(e => {
+        console.error('Error caching topics: ', e);
+    })
+
+    // The producer can be used as usual.
+    await producer.send({ topic: 'test-topic', messages: [{ value: 'Hello!' }] });
+
     // Disconnect the producer and admin clients in the correct order.
     await admin.disconnect();
     await producer.disconnect();

examples/node-rdkafka/dependent_admin.js renamed to examples/node-rdkafka/dependent-admin.js

@@ -9,7 +9,7 @@ function adminFromProducer() {
         'dr_msg_cb': true,
     });
 
-    const createAdminAndListTopics = () => {
+    const createAdminAndListAndDescribeTopics = (done) => {
         // Create an admin client from the producer, which must be connected.
         // Thus, this is called from the producer's 'ready' event.
         const admin = Kafka.AdminClient.createFrom(producer);
@@ -21,7 +21,26 @@ function adminFromProducer() {
                 return;
             }
             console.log("Topics: ", topics);
-            admin.disconnect();
+
+            // A common use case for the dependent admin client is to make sure the topic
+            // is cached before producing to it. This avoids delay in sending the first
+            // message to any topic. Using the admin client linked to the producer allows
+            // us to do this, by calling `describeTopics` before we produce.
+            // Here, we cache all possible topics, but it's advisable to only cache the
+            // topics you are going to produce to (if you know it in advance),
+            // and avoid calling listTopics().
+            // Once a topic is cached, it will stay cached for `metadata.max.age.ms`,
+            // which is 15 minutes by default, after which it will be removed if
+            // it has not been produced to.
+            admin.describeTopics(topics, null, (err, topicDescriptions) => {
+                if (err) {
+                    console.error(err);
+                    return;
+                }
+                console.log("Topic descriptions fetched successfully");
+                admin.disconnect();
+                done();
+            });
         });
     };
 
@@ -32,10 +51,11 @@ function adminFromProducer() {
         producer.setPollInterval(100);
 
         // After the producer is ready, it can be used to create an admin client.
-        createAdminAndListTopics();
+        createAdminAndListAndDescribeTopics(() => {
+            // The producer can also be used normally to produce messages.
+            producer.produce('test-topic', null, Buffer.from('Hello World!'), null, Date.now());
+        });
 
-        // It can also be used normally to produce messages.
-        producer.produce('test-topic', null, Buffer.from('Hello World!'), null, Date.now());
     });
 
     producer.on('event.error', (err) => {