Patch

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

Author: larryliu0820

examples/models/llama2/export_llama_lib.py

@@ -29,7 +29,7 @@
     Transformer,
 )
 from executorch.exir.backend.backend_details import CompileSpec
-
+from executorch.exir.passes import *
 from executorch.sdk.etrecord import generate_etrecord
 from executorch.util.activation_memory_profiler import generate_memory_trace
 from sentencepiece import SentencePieceProcessor
@@ -539,7 +539,7 @@ def _prepare_for_llama_export(modelname: str, args) -> LlamaEdgeManager:
         bitwidth = int(bitwidth)
         transforms.append(
             lambda model: EmbeddingQuantHandler(
-                model, bitwidth=bitwidth, group_size=group_size
+                model, bitwidth=bitwidth, group_size=group_size, packed=(bitwidth==4),
             ).quantized_model()
         )
 

examples/models/llama2/quantize.py

@@ -436,18 +436,10 @@ def __init__(
     @torch.no_grad()
     def forward(self, indices: torch.Tensor) -> torch.Tensor:
         if not self.packed:  # 8bit
-<<<<<<< HEAD
             return torch.ops.quantized_decomposed.embedding_byte.dtype(
                 self.weight, self.scales, None, 0, 0, indices, dtype=self.dtype
             )
         else:  # 4bit packed
             return torch.ops.quantized_decomposed.embedding_4bit.dtype(
-=======
-            return torch.ops.llama_quantized.DEPRECATED_DO_NOT_USE_embedding_byte.dtype(
-                self.weight, self.scales, None, 0, 0, indices, dtype=self.dtype
-            )
-        else:  # 4bit packed
-            return torch.ops.llama_quantized.embedding_4bit.dtype(
->>>>>>> 6b3b7228c (4b embedding quantizer (#3081))
                 self.weight, self.scales, None, 0, 0, indices, dtype=self.dtype
             )