0002-sentencepiece-tokenization-not-supported.patch 2.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051
  1. From bd41a985345f17306e472f7d825d30b3e3d0baba Mon Sep 17 00:00:00 2001
  2. From: Robert Ogden <robertogden@chromium.org>
  3. Date: Wed, 25 May 2022 10:52:49 -0700
  4. Subject: [PATCH 2/9] sentencepiece tokenization not supported
  5. ---
  6. .../cc/text/tokenizers/tokenizer_utils.cc | 14 ++++----------
  7. 1 file changed, 4 insertions(+), 10 deletions(-)
  8. diff --git a/third_party/tflite_support/src/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc b/third_party/tflite_support/src/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc
  9. index 9abca9691f058..28f0137f54278 100644
  10. --- a/third_party/tflite_support/src/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc
  11. +++ b/third_party/tflite_support/src/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc
  12. @@ -20,7 +20,6 @@ limitations under the License.
  13. #include "tensorflow_lite_support/cc/port/status_macros.h"
  14. #include "tensorflow_lite_support/cc/text/tokenizers/bert_tokenizer.h"
  15. #include "tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h"
  16. -#include "tensorflow_lite_support/cc/text/tokenizers/sentencepiece_tokenizer.h"
  17. #include "tensorflow_lite_support/metadata/metadata_schema_generated.h"
  18. namespace tflite {
  19. @@ -29,7 +28,6 @@ namespace text {
  20. namespace tokenizer {
  21. using ::tflite::ProcessUnit;
  22. -using ::tflite::SentencePieceTokenizerOptions;
  23. using ::tflite::support::CreateStatusWithPayload;
  24. using ::tflite::support::StatusOr;
  25. using ::tflite::support::TfLiteSupportStatus;
  26. @@ -74,14 +72,10 @@ StatusOr<std::unique_ptr<Tokenizer>> CreateTokenizerFromProcessUnit(
  27. vocab_buffer.size());
  28. }
  29. case ProcessUnitOptions_SentencePieceTokenizerOptions: {
  30. - const tflite::SentencePieceTokenizerOptions* options =
  31. - tokenizer_process_unit->options_as<SentencePieceTokenizerOptions>();
  32. - ASSIGN_OR_RETURN(absl::string_view model_buffer,
  33. - CheckAndLoadFirstAssociatedFile(
  34. - options->sentencePiece_model(), metadata_extractor));
  35. - // TODO(b/160647204): Extract sentence piece model vocabulary
  36. - return absl::make_unique<SentencePieceTokenizer>(model_buffer.data(),
  37. - model_buffer.size());
  38. + return CreateStatusWithPayload(
  39. + absl::StatusCode::kInvalidArgument,
  40. + "Chromium does not support sentencepiece tokenization",
  41. + TfLiteSupportStatus::kMetadataInvalidTokenizerError);
  42. }
  43. case ProcessUnitOptions_RegexTokenizerOptions: {
  44. const tflite::RegexTokenizerOptions* options =
  45. --
  46. 2.36.1.124.g0e6072fb45-goog