o Yi7@sdZddlZddlmZmZddlmZmZddlmZddl m Z ddl m Z dd l m Zdd l mZed Zgd ZGd ddZdedejjjdefddZdedefddZGdddZdS)z)[Experimental] Text Only Local Tokenizer.N)AnyIterable)OptionalUnion)sentencepiece_model_pb2)_common)_local_tokenizer_loader) _transformers)typeszgoogle_genai.local_tokenizer)_parse_hex_byte_token_str_to_bytesLocalTokenizer_TextsAccumulatorc@s&eZdZdZd)ddZdeefddZdeej ddfd d Z d ej ddfd d Z dej ddfddZ dejdejfddZdeejddfddZdeejddfddZdejddfddZdejdejfddZd ejdejfd!d"Zd#eeefdeeeffd$d%Zd&edefd'd(ZdS)*raAccumulates countable texts from `Content` and `Tool` objects. This class is responsible for traversing complex `Content` and `Tool` objects and extracting all the text content that should be included when calculating token counts. A key feature of this class is its ability to detect unsupported fields in `Content` objects. If a user provides a `Content` object with fields that this local tokenizer doesn't recognize (e.g., new fields added in a future API update), this class will log a warning. The detection mechanism for `Content` objects works by recursively building a "counted" version of the input object. This "counted" object only contains the data that was successfully processed and added to the text list for tokenization. After traversing the input, the original `Content` object is compared to the "counted" object. If they don't match, it signifies the presence of unsupported fields, and a warning is logged. returnNcCs g|_dSN_textsselfrV/home/kim/smarthome/.venv/lib/python3.10/site-packages/google/genai/local_tokenizer.py__init__;s z_TextsAccumulator.__init__cCs|jSrrrrrr get_texts>sz_TextsAccumulator.get_textscontentscC|D]}||qdSr) add_content)rrcontentrrr add_contentsA z_TextsAccumulator.add_contentsrcCstjg|jd}|jrg|jD]X}|jdusJt}|jdus%|jdur)td|jdur2|j|_|j durA| |j |j |_ |j durP| |j |j |_ |j dur`|j |_ |j|j |j|q|jdd|jddkrtd|d|ddSdS)N)partsrolez6LocalTokenizers do not support non-text content types.T)Z exclude_nonezHContent contains unsupported types for token counting. Supported fields z. Got .)r Contentr!r ZPart file_dataZ inline_data ValueErrorZvideo_metadata function_calladd_function_callfunction_responseadd_function_responsetextrappendZ model_dumploggerwarning)rrZcounted_contentpartZ counted_partrrrrEs@       z_TextsAccumulator.add_contentr&cCsB|jr |j|jtj|jd}|jr||j}||_dSdS)zProcesses a function call and adds relevant text to the accumulator. Args: function_call: The function call to process. )nameN)r/rr+r FunctionCallargs_dict_traverse)rr&Zcounted_function_callZ counted_argsrrrr'ds  z#_TextsAccumulator.add_function_calltoolcCsHtjgd}|jr"|jD]}||}|jdurg|_|j|q |S)N)function_declarations)r Toolr4_function_declaration_traverser+)rr3Z counted_toolfunction_declarationcounted_function_declarationrrradd_toolqs   z_TextsAccumulator.add_tooltoolscCrr)r9)rr:r3rrr add_tools~rz_TextsAccumulator.add_toolsfunction_responsescCrr)r))rr<r(rrradd_function_responsess z(_TextsAccumulator.add_function_responsesr(cCsDt}|jr|j|j|j|_|jr ||j}||_dSdSr)r FunctionResponser/rr+responser2)rr(Zcounted_function_responsecounted_responserrrr)s  z'_TextsAccumulator.add_function_responser7cCstt}|jr|j|j|j|_|jr |j|j|j|_|jr,||j}||_|jr8||j}||_|Sr) r FunctionDeclarationr/rr+ description parameters add_schemar?)rr7r8Zcounted_parametersr@rrrr6s  z0_TextsAccumulator._function_declaration_traverseschemac Cs*t}|jr |j|_|jr|j|_|jdur|j|_|jr)|j|j|j|_|jr7|j|j|j|_|j rE|j |j |j |_ |j rS|j |j |j |_ |j rZ|j |_ |j rf||j }||_ |jri}|j D]\}}|j|||}|||<qp||_|jr||j}||_|S)zProcesses a schema and adds relevant text to the accumulator. Args: schema: The schema to process. Returns: The new schema object with only countable fields. N)r Schematypetitledefaultformatrr+rBenumextendrequiredZproperty_orderingitemsrD propertiesZexample _any_traverse) rrEZcounted_schemaZcounted_schema_itemsdkeyvalueZ counted_valueZcounted_schema_examplerrrrDsF       z_TextsAccumulator.add_schemarQcCs<i}|jt||D] \}}||||<q|S)zProcesses a dict and adds relevant text to the accumulator. Args: d: The dict to process. Returns: The new dict object with only countable fields. )rrLlistkeysrNrP)rrQZ counted_dictrRvalrrrr2s  z _TextsAccumulator._dict_traverserScsNt|tr j||St|tr|St|tr%fdd|DS|S)zProcesses a value and adds relevant text to the accumulator. Args: value: The value to process. Returns: The new value with only countable fields. csg|]}|qSr)rP).0itemrrr sz3_TextsAccumulator._any_traverse..) isinstancestrrr+dictr2rT)rrSrrrrPs    z_TextsAccumulator._any_traverse)rN)__name__ __module__ __qualname____doc__rrr[rr r#rrr0r'r5r9r;r>r=r)rAr6rFrDr\rr2rPrrrrr's6       "-rtokenrGrcCs4|tjjjjkrt|jdddS|dddS)Nrbig)length byteorderu▁ zutf-8) r ModelProto SentencePieceTypeZBYTEr to_bytesreplaceencode)rarGrrrr sr cCst|dkr td||dr|dstd|z t|ddd}Wnty7td |w|d krCtd ||S) zParses a hex byte string of the form '<0xXX>' and returns the integer value. Raises ValueError if the input is malformed or the byte value is invalid. zInvalid byte length: z<0x>zInvalid byte format: zInvalid hex value: zByte value out of range: )lenr% startswithendswithint)rarVrrrr s  r c@seZdZdZdefddZeddddee j e j fd e e j d e jfd d Zeddee j e j fd e jfd dZdS)ra[Experimental] Text Only Local Tokenizer. This class provides a local tokenizer for text only token counting. LIMITATIONS: - Only supports text based tokenization and no multimodal tokenization. - Forward compatibility depends on the open-source tokenizer models for future Gemini versions. - For token counting of tools and response schemas, the `LocalTokenizer` only supports `types.Tool` and `types.Schema` objects. Python functions or Pydantic models cannot be passed directly. model_namecCs,t||_t|j|_t|j|_dSr)loaderZget_tokenizer_nameZ_tokenizer_nameZload_model_proto _model_protoZget_sentencepiece _tokenizer)rrvrrrr$s zLocalTokenizer.__init__zThe SDK's local tokenizer implementation is experimental and may change in the future. It only supports text based tokenization.N)configrrzrcCst|}t}tj|pi}|||jr||j|j r,|j j r,| |j j |j r9|t|j g|j t|}tjtdd|DdS)aCounts the number of tokens in a given text. Args: contents: The contents to tokenize. config: The configuration for counting tokens. Returns: A `CountTokensResult` containing the total number of tokens. Usage: .. code-block:: python from google import genai tokenizer = genai.LocalTokenizer(model_name='gemini-2.0-flash-001') result = tokenizer.count_tokens("What is your name?") print(result) # total_tokens=5 css|]}t|VqdSr)rr)rWtokensrrr Rsz.LocalTokenizer.count_tokens..)Z total_tokens)t t_contentsrr ZCountTokensConfigZmodel_validaterr:r;Zgeneration_configZresponse_schemarDZsystem_instructionryrkrTrCountTokensResultsum)rrrzprocessed_contentstext_accumulatorZ tokens_listrrr count_tokens)s   zLocalTokenizer.count_tokensc st|}t}|D]}||q j|}g}|D]}|jr/|jD]}||j q&qg}t ||D]\} } |t j dd| j Dfdd| j D| dq7t j|dS)a,Computes the tokens ids and string pieces in the input. Args: contents: The contents to tokenize. Returns: A `ComputeTokensResult` containing the token information. Usage: .. code-block:: python from google import genai tokenizer = genai.LocalTokenizer(model_name='gemini-2.0-flash-001') result = tokenizer.compute_tokens("What is your name?") print(result) # tokens_info=[TokensInfo(token_ids=[279, 329, 1313, 2508, 13], tokens=[b' What', b' is', b' your', b' name', b'?'], role='user')] cSsg|]}|jqSr)idrWpiecerrrrYsz1LocalTokenizer.compute_tokens..cs$g|]}t|jjj|jjqSr)r rrxpiecesrrGrrrrrYs )Z token_idsr{r!)Z tokens_info)r}r~rrryZEncodeAsImmutableProtorr r+r!zipr Z TokensInforComputeTokensResult) rrrrrZ tokens_protosZroles_Z token_infosZ tokens_protor!rrrcompute_tokensUs2     zLocalTokenizer.compute_tokens)r]r^r_r`r[rrZexperimental_warningrr ZContentListUnionZContentListUnionDictrZCountTokensConfigOrDictrrrrrrrrrs,  (r)r`loggingtypingrrrrZ sentencepiecerrr rwr r}r getLoggerr,__all__rr[rfrgrhbytesr rur rrrrrs,      Q