[Do not merge] Hosted handler for MPT

.github/workflows/codeql-analysis.yml

@@ -14,6 +14,9 @@ name: "CodeQL"
 on:
   push:
     branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
   schedule:
     - cron: '0 9 * * 1'  # Every Monday at 09:00 (9:00 AM)
 

examples/inference-deployments/mpt/mpt_7b_ft_handler.py

@@ -307,11 +307,14 @@ def predict(self, model_requests: List[Dict]) -> List[str]:
         start_lengths = torch.IntTensor(start_lengths)
         tokens_batch = self.model(start_ids, start_lengths, **generate_kwargs)
         outputs = []
-        for tokens in tokens_batch:
+        for i, tokens in enumerate(tokens_batch):
             for beam_id in range(generate_kwargs['beam_width']):
-                # Do not exclude context input from the output
-                # token = tokens[beam_id][start_lengths[i]:]
-                token = tokens[beam_id]
+                # Exclude context input from the output
+                token = tokens[beam_id][start_lengths[i]:]
+
+                # Do this to exclude context input from the output
+                # token = tokens[beam_id]
+
                 # stop at end_id; This is the same as eos_token_id
                 token = token[token != self.end_id]
                 output = self.tokenizer.decode(token)