org.bytedeco.sentencepiece

Class SentencePieceProcessor

java.lang.Object

org.bytedeco.javacpp.Pointer

org.bytedeco.sentencepiece.SentencePieceProcessor

All Implemented Interfaces:

AutoCloseable

@Namespace(value="sentencepiece")
 @NoOffset
 @Properties(inherit=sentencepiece.class)
public class SentencePieceProcessor
extends Pointer

Nested Class Summary

Nested classes/interfaces inherited from class org.bytedeco.javacpp.Pointer

Pointer.CustomDeallocator, Pointer.Deallocator, Pointer.NativeDeallocator, Pointer.ReferenceCounter

Field Summary

Fields inherited from class org.bytedeco.javacpp.Pointer

address, capacity, limit, position

Constructor Summary

Constructors

Constructor and Description
SentencePieceProcessor()
SentencePieceProcessor(long size) Native array allocator.
SentencePieceProcessor(Pointer p) Pointer cast constructor.

Method Summary

Modifier and Type	Method and Description
int	bos_id()
float	CalculateEntropy(String text, float alpha)
Status	CalculateEntropy(String input, float alpha, float[] entropy)
Status	CalculateEntropy(String input, float alpha, FloatBuffer entropy)
Status	CalculateEntropy(String input, float alpha, FloatPointer entropy)
Status	Decode(IntVector ids, BytePointer detokenized)
Status	Decode(IntVector ids, SentencePieceText spt)
Status	Decode(StringVector pieces, BytePointer detokenized)
Status	Decode(StringVector pieces, SentencePieceText spt)
String	DecodeIds(IntVector ids)
ImmutableSentencePieceText	DecodeIdsAsImmutableProto(IntVector ids)
String	DecodeIdsAsSerializedProto(IntVector ids)
String	DecodePieces(StringVector pieces)
ImmutableSentencePieceText	DecodePiecesAsImmutableProto(StringVector pieces)
String	DecodePiecesAsSerializedProto(StringVector pieces)
Status	Encode(String input, IntVector ids)
Status	Encode(String input, SentencePieceText spt)
Status	Encode(String input, StringVector pieces)
IntVector	EncodeAsIds(String input)
ImmutableSentencePieceText	EncodeAsImmutableProto(String input)
StringVector	EncodeAsPieces(String input)
String	EncodeAsSerializedProto(String input)
int	eos_id()
int	GetPieceSize()
SentencePieceProcessor	getPointer(long i)
float	GetScore(int id)
String	IdToPiece(int id)
boolean	IsByte(int id)
boolean	IsControl(int id)
boolean	IsUnknown(int id)
boolean	IsUnused(int id)
Status	Load(ModelProto model_proto)
Status	Load(String filename)
Status	LoadFromSerializedProto(String serialized)
void	LoadOrDie(String filename)
Status	LoadVocabulary(String filename, int threshold)
ModelProto	model_proto()
NormalizerSpec	mutable_normalizer_spec()
Status	NBestEncode(String input, int nbest_size, IntVector ids)
Status	NBestEncode(String input, int nbest_size, NBestSentencePieceText nbest_spt)
Status	NBestEncode(String input, int nbest_size, StringVector pieces)
IntVector	NBestEncodeAsIds(String input, int nbest_size)
ImmutableNBestSentencePieceText	NBestEncodeAsImmutableProto(String input, int nbest_size)
StringVector	NBestEncodeAsPieces(String input, int nbest_size)
String	NBestEncodeAsSerializedProto(String input, int nbest_size)
String	Normalize(String input)
Status	Normalize(String input, BytePointer normalized)
Status	Normalize(String input, BytePointer normalized, SizeTPointer norm_to_orig)
int	pad_id()
int	PieceToId(String piece)
SentencePieceProcessor	position(long position)
Status	ResetVocabulary()
Status	SampleEncode(String input, int nbest_size, float alpha, IntVector ids)
Status	SampleEncode(String input, int nbest_size, float alpha, SentencePieceText spt)
Status	SampleEncode(String input, int nbest_size, float alpha, StringVector pieces)
Status	SampleEncodeAndScore(String input, int num_samples, float alpha, boolean wor, boolean include_best, IntVectorFloatPairVector ids)
Status	SampleEncodeAndScore(String input, int num_samples, float alpha, boolean wor, boolean include_best, NBestSentencePieceText samples_spt)
Status	SampleEncodeAndScore(String input, int num_samples, float alpha, boolean wor, boolean include_best, StringVectorFloatPairVector pieces)
IntVectorFloatPairVector	SampleEncodeAndScoreAsIds(String input, int num_samples, float alpha, boolean wor, boolean include_best)
ImmutableNBestSentencePieceText	SampleEncodeAndScoreAsImmutableProto(String input, int num_samples, float alpha, boolean wor, boolean include_best)
StringVectorFloatPairVector	SampleEncodeAndScoreAsPieces(String input, int num_samples, float alpha, boolean wor, boolean include_best)
String	SampleEncodeAndScoreAsSerializedProto(String input, int num_samples, float alpha, boolean wor, boolean include_best)
IntVector	SampleEncodeAsIds(String input, int nbest_size, float alpha)
ImmutableSentencePieceText	SampleEncodeAsImmutableProto(String input, int nbest_size, float alpha)
StringVector	SampleEncodeAsPieces(String input, int nbest_size, float alpha)
String	SampleEncodeAsSerializedProto(String input, int nbest_size, float alpha)
String	serialized_model_proto()
Status	SetDecodeExtraOptions(String extra_option)
Status	SetEncodeExtraOptions(String extra_option)
Status	status()
int	unk_id()

Methods inherited from class org.bytedeco.javacpp.Pointer

address, asBuffer, asByteBuffer, availablePhysicalBytes, calloc, capacity, capacity, close, deallocate, deallocate, deallocateReferences, deallocator, deallocator, equals, fill, formatBytes, free, getDirectBufferAddress, getPointer, getPointer, getPointer, hashCode, interruptDeallocatorThread, isNull, isNull, limit, limit, malloc, maxBytes, maxPhysicalBytes, memchr, memcmp, memcpy, memmove, memset, offsetAddress, offsetof, offsetof, parseBytes, physicalBytes, physicalBytesInaccurate, position, put, realloc, referenceCount, releaseReference, retainReference, setNull, sizeof, sizeof, toString, totalBytes, totalCount, totalPhysicalBytes, withDeallocator, zero

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Constructor Detail

SentencePieceProcessor

public SentencePieceProcessor(Pointer p)

Pointer cast constructor. Invokes Pointer(Pointer).

SentencePieceProcessor

public SentencePieceProcessor(long size)

Native array allocator. Access with Pointer.position(long).

SentencePieceProcessor

public SentencePieceProcessor()

Method Detail

position

public SentencePieceProcessor position(long position)

Overrides:

position in class Pointer

getPointer

public SentencePieceProcessor getPointer(long i)

Overrides:

getPointer in class Pointer

Load

@ByVal
public Status Load(@StdString
                           String filename)

LoadOrDie

public void LoadOrDie(@StdString
                      String filename)

Load

@ByVal
public Status Load(@Const @ByRef
                           ModelProto model_proto)

LoadFromSerializedProto

@ByVal
public Status LoadFromSerializedProto(@StdString
                                              String serialized)

status

@ByVal
public Status status()

SetEncodeExtraOptions

@ByVal
public Status SetEncodeExtraOptions(@StdString
                                            String extra_option)

SetDecodeExtraOptions

@ByVal
public Status SetDecodeExtraOptions(@StdString
                                            String extra_option)

ResetVocabulary

@ByVal
public Status ResetVocabulary()

LoadVocabulary

@ByVal
public Status LoadVocabulary(@StdString
                                     String filename,
                                     int threshold)

Encode

@ByVal
public Status Encode(@StdString
                             String input,
                             StringVector pieces)

Encode

@ByVal
public Status Encode(@StdString
                             String input,
                             IntVector ids)

Decode

@ByVal
public Status Decode(@Const @ByRef
                             StringVector pieces,
                             @StdString @Cast(value={"char*","std::string*"})
                             BytePointer detokenized)

Decode

@ByVal
public Status Decode(@Const @ByRef
                             IntVector ids,
                             @StdString @Cast(value={"char*","std::string*"})
                             BytePointer detokenized)

NBestEncode

@ByVal
public Status NBestEncode(@StdString
                                  String input,
                                  int nbest_size,
                                  @StdVector
                                  StringVector pieces)

NBestEncode

@ByVal
public Status NBestEncode(@StdString
                                  String input,
                                  int nbest_size,
                                  @StdVector
                                  IntVector ids)

SampleEncode

@ByVal
public Status SampleEncode(@StdString
                                   String input,
                                   int nbest_size,
                                   float alpha,
                                   StringVector pieces)

SampleEncode

@ByVal
public Status SampleEncode(@StdString
                                   String input,
                                   int nbest_size,
                                   float alpha,
                                   IntVector ids)

SampleEncodeAndScore

@ByVal
public Status SampleEncodeAndScore(@StdString
                                           String input,
                                           int num_samples,
                                           float alpha,
                                           @Cast(value="bool")
                                           boolean wor,
                                           @Cast(value="bool")
                                           boolean include_best,
                                           StringVectorFloatPairVector pieces)

SampleEncodeAndScore

@ByVal
public Status SampleEncodeAndScore(@StdString
                                           String input,
                                           int num_samples,
                                           float alpha,
                                           @Cast(value="bool")
                                           boolean wor,
                                           @Cast(value="bool")
                                           boolean include_best,
                                           IntVectorFloatPairVector ids)

CalculateEntropy

@ByVal
public Status CalculateEntropy(@StdString
                                       String input,
                                       float alpha,
                                       FloatPointer entropy)

CalculateEntropy

@ByVal
public Status CalculateEntropy(@StdString
                                       String input,
                                       float alpha,
                                       FloatBuffer entropy)

CalculateEntropy

@ByVal
public Status CalculateEntropy(@StdString
                                       String input,
                                       float alpha,
                                       float[] entropy)

Encode

@ByVal
public Status Encode(@StdString
                             String input,
                             SentencePieceText spt)

NBestEncode

@ByVal
public Status NBestEncode(@StdString
                                  String input,
                                  int nbest_size,
                                  NBestSentencePieceText nbest_spt)

SampleEncode

@ByVal
public Status SampleEncode(@StdString
                                   String input,
                                   int nbest_size,
                                   float alpha,
                                   SentencePieceText spt)

SampleEncodeAndScore

@ByVal
public Status SampleEncodeAndScore(@StdString
                                           String input,
                                           int num_samples,
                                           float alpha,
                                           @Cast(value="bool")
                                           boolean wor,
                                           @Cast(value="bool")
                                           boolean include_best,
                                           NBestSentencePieceText samples_spt)

Decode

@ByVal
public Status Decode(@Const @ByRef
                             StringVector pieces,
                             SentencePieceText spt)

Decode

@ByVal
public Status Decode(@Const @ByRef
                             IntVector ids,
                             SentencePieceText spt)

EncodeAsPieces

@ByVal
public StringVector EncodeAsPieces(@StdString
                                           String input)

EncodeAsIds

@ByVal
public IntVector EncodeAsIds(@StdString
                                     String input)

NBestEncodeAsPieces

@StdVector
public StringVector NBestEncodeAsPieces(@StdString
                                                    String input,
                                                    int nbest_size)

NBestEncodeAsIds

@StdVector
public IntVector NBestEncodeAsIds(@StdString
                                              String input,
                                              int nbest_size)

SampleEncodeAsPieces

@ByVal
public StringVector SampleEncodeAsPieces(@StdString
                                                 String input,
                                                 int nbest_size,
                                                 float alpha)

SampleEncodeAsIds

@ByVal
public IntVector SampleEncodeAsIds(@StdString
                                           String input,
                                           int nbest_size,
                                           float alpha)

SampleEncodeAndScoreAsPieces

@ByVal
public StringVectorFloatPairVector SampleEncodeAndScoreAsPieces(@StdString
                                                                        String input,
                                                                        int num_samples,
                                                                        float alpha,
                                                                        @Cast(value="bool")
                                                                        boolean wor,
                                                                        @Cast(value="bool")
                                                                        boolean include_best)

SampleEncodeAndScoreAsIds

@ByVal
public IntVectorFloatPairVector SampleEncodeAndScoreAsIds(@StdString
                                                                  String input,
                                                                  int num_samples,
                                                                  float alpha,
                                                                  @Cast(value="bool")
                                                                  boolean wor,
                                                                  @Cast(value="bool")
                                                                  boolean include_best)

DecodePieces

@StdString
public String DecodePieces(@Const @ByRef
                                       StringVector pieces)

DecodeIds

@StdString
public String DecodeIds(@Const @ByRef
                                    IntVector ids)

CalculateEntropy

public float CalculateEntropy(@StdString
                              String text,
                              float alpha)

EncodeAsSerializedProto

@StdString
public String EncodeAsSerializedProto(@StdString
                                                  String input)

SampleEncodeAsSerializedProto

@StdString
public String SampleEncodeAsSerializedProto(@StdString
                                                        String input,
                                                        int nbest_size,
                                                        float alpha)

NBestEncodeAsSerializedProto

@StdString
public String NBestEncodeAsSerializedProto(@StdString
                                                       String input,
                                                       int nbest_size)

SampleEncodeAndScoreAsSerializedProto

@StdString
public String SampleEncodeAndScoreAsSerializedProto(@StdString
                                                                String input,
                                                                int num_samples,
                                                                float alpha,
                                                                @Cast(value="bool")
                                                                boolean wor,
                                                                @Cast(value="bool")
                                                                boolean include_best)

DecodePiecesAsSerializedProto

@StdString
public String DecodePiecesAsSerializedProto(@Const @ByRef
                                                        StringVector pieces)

DecodeIdsAsSerializedProto

@StdString
public String DecodeIdsAsSerializedProto(@Const @ByRef
                                                     IntVector ids)

EncodeAsImmutableProto

@ByVal
public ImmutableSentencePieceText EncodeAsImmutableProto(@StdString
                                                                 String input)

SampleEncodeAsImmutableProto

@ByVal
public ImmutableSentencePieceText SampleEncodeAsImmutableProto(@StdString
                                                                       String input,
                                                                       int nbest_size,
                                                                       float alpha)

NBestEncodeAsImmutableProto

@ByVal
public ImmutableNBestSentencePieceText NBestEncodeAsImmutableProto(@StdString
                                                                           String input,
                                                                           int nbest_size)

SampleEncodeAndScoreAsImmutableProto

@ByVal
public ImmutableNBestSentencePieceText SampleEncodeAndScoreAsImmutableProto(@StdString
                                                                                    String input,
                                                                                    int num_samples,
                                                                                    float alpha,
                                                                                    @Cast(value="bool")
                                                                                    boolean wor,
                                                                                    @Cast(value="bool")
                                                                                    boolean include_best)

DecodePiecesAsImmutableProto

@ByVal
public ImmutableSentencePieceText DecodePiecesAsImmutableProto(@Const @ByRef
                                                                       StringVector pieces)

DecodeIdsAsImmutableProto

@ByVal
public ImmutableSentencePieceText DecodeIdsAsImmutableProto(@Const @ByRef
                                                                    IntVector ids)

Normalize

@ByVal
public Status Normalize(@StdString
                                String input,
                                @StdString @Cast(value={"char*","std::string*"})
                                BytePointer normalized)

Normalize

@ByVal
public Status Normalize(@StdString
                                String input,
                                @StdString @Cast(value={"char*","std::string*"})
                                BytePointer normalized,
                                @Cast(value="size_t*") @StdVector
                                SizeTPointer norm_to_orig)

Normalize

@StdString
public String Normalize(@StdString
                                    String input)

GetPieceSize

public int GetPieceSize()

PieceToId

public int PieceToId(@StdString
                     String piece)

IdToPiece

@StdString
public String IdToPiece(int id)

GetScore

public float GetScore(int id)

IsUnknown

@Cast(value="bool")
public boolean IsUnknown(int id)

IsControl

@Cast(value="bool")
public boolean IsControl(int id)

IsUnused

@Cast(value="bool")
public boolean IsUnused(int id)

IsByte

@Cast(value="bool")
public boolean IsByte(int id)

unk_id

public int unk_id()

bos_id

public int bos_id()

eos_id

public int eos_id()

pad_id

public int pad_id()

model_proto

@Const
 @ByRef
public ModelProto model_proto()

serialized_model_proto

@StdString
public String serialized_model_proto()

mutable_normalizer_spec

public NormalizerSpec mutable_normalizer_spec()