U 9%eK@sddlmZddlmZmZmZmZddlZddl Z ddl m Z ddl m Z ddlmZmZmZdd lmZdd lmZmZmZerdd lmZdd lmZdd lmZeeZ erddl!Z!ddl"m#Z#m$Z$ddZ%dddZ&ddZ'ddZ(GdddeZ)ddZ*dS)) defaultdict) TYPE_CHECKINGDictOptionalUnionN) ModelCard)PreTrainedTokenizer)is_torch_availableis_torchaudio_availablelogging) ffmpeg_read)ArgumentHandler ChunkPipelineinfer_framework_load_model)BeamSearchDecoderCTC)SequenceFeatureExtractor)PreTrainedModel)MODEL_FOR_CTC_MAPPING_NAMES(MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMEScCsdg}|D]V\}}}tt||}tt|||}tt|||}|||f}||q|S)z Rescales the stride values from audio space to tokens/logits space. (160_000, 16_000, 16_000) -> (2000, 200, 200) for instance. )introundappend)strideratioZ new_stridesZinput_nleftrightZtoken_nZ new_striderr/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/pipelines/automatic_speech_recognition.pyrescale_stride)s   r Tccs4|jd}|||}td||D] } | |} || | } || |jdd} |dk r`| j|d} | dkrldn|} |dkr| |kn| |k}|rdn|}| jd}|| |f}d| kr| djd}nd| kr| djd}|| jdkr|r||}t|g|d}| jd| kr$||d| V|r"q0q"dS) Nrpt sampling_rateZreturn_tensorsdtypeinput_features input_values)is_lastr)shaperanger#tor )inputsfeature_extractor chunk_len stride_left stride_rightrescaler%Z inputs_lenstepZchunk_start_idxZ chunk_end_idxchunk processedZ _stride_leftr)Z _stride_rightrZ processed_lenrrrr chunk_iter=s0       r6c st|}t|fddt|dD}d}t|D]R}tD]D}||||krB|||d}|||d|d<||krB|}qBq6t|}|dkrt||kdd|nd}|dkrt||kdd|nd} || |fS)Ncsg|]}dgdqS)rr r).0_Z seq_len_rightrr ]sz6_fast_find_longest_common_sequence..r rr')lenr+nparrayZargwhere) Z sequence_leftZsequence_rightZ seq_len_leftcounterlongestijZprevious_counter index_left index_rightrr9r"_fast_find_longest_common_sequenceZs   &&rDc sfdd|ddD}|ddD]}fdd|dD}d}d}tdt|dD]\}|d}tt|| dt|d|k} | ||} | dkrb| |krb|}| }qb|||dq*t|S)Ncsg|]}|jkr|qSrZall_special_idsr7Ztok_id tokenizerrrr:us z1_find_longest_common_sequence..rr csg|]}|jkr|qSrrErFrGrrr:ws gg@)tolistr+r;r<sumr=extend) sequencesrHsequenceZnew_seqZ new_sequenceindexZmax_r@ZepsmatchesZmatchingrrGr_find_longest_common_sequencens, rPcseZdZdZddedefeeeedefeeeeee ee dfeeed fe d d d Z ee jeefd fdd ZdddZdddZdddZdeedddZZS)"AutomaticSpeechRecognitionPipelinea} Pipeline that aims at extracting spoken text contained within some audio. The input can be either a raw waveform or a audio file. In case of the audio file, ffmpeg should be installed for to support multiple audio formats Example: ```python >>> from transformers import pipeline >>> transcriber = pipeline(model="openai/whisper-base") >>> transcriber("https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/1.flac") {'text': ' He hoped there would be stew for dinner, turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick, peppered flour-fatten sauce.'} ``` Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) Arguments: model ([`PreTrainedModel`] or [`TFPreTrainedModel`]): The model that will be used by the pipeline to make predictions. This needs to be a model inheriting from [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for TensorFlow. tokenizer ([`PreTrainedTokenizer`]): The tokenizer that will be used by the pipeline to encode data for the model. This object inherits from [`PreTrainedTokenizer`]. feature_extractor ([`SequenceFeatureExtractor`]): The feature extractor that will be used by the pipeline to encode waveform for the model. chunk_length_s (`float`, *optional*, defaults to 0): The input length for in each chunk. If `chunk_length_s = 0` then chunking is disabled (default). For more information on how to effectively use `chunk_length_s`, please have a look at the [ASR chunking blog post](https://huggingface.co/blog/asr-chunking). stride_length_s (`float`, *optional*, defaults to `chunk_length_s / 6`): The length of stride on the left and right of each chunk. Used only with `chunk_length_s > 0`. This enables the model to *see* more context and infer letters better than without this context but the pipeline discards the stride bits at the end to make the final reconstitution as perfect as possible. For more information on how to effectively use `stride_length_s`, please have a look at the [ASR chunking blog post](https://huggingface.co/blog/asr-chunking). framework (`str`, *optional*): The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework must be installed. If no framework is specified, will default to the one currently installed. If no framework is specified and both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model is provided. device (Union[`int`, `torch.device`], *optional*): Device ordinal for CPU/GPU supports. Setting this to `None` will leverage CPU, a positive will run the model on the associated CUDA device id. decoder (`pyctcdecode.BeamSearchDecoderCTC`, *optional*): [PyCTCDecode's BeamSearchDecoderCTC](https://github.com/kensho-technologies/pyctcdecode/blob/2fd33dc37c4111417e08d89ccd23d28e9b308d19/pyctcdecode/decoder.py#L180) can be passed for language model boosted decoding. See [`Wav2Vec2ProcessorWithLM`] for more information. NFrrrz torch.devicez torch.dtype) modelr.rHdecoder modelcard frameworktask args_parserdevice torch_dtype binary_outputc Ksf|dkrt||jd\}}||_||_||_||_||_||_t|jdd} | dk rd| dk rdt d|jdkrvt d| dk rt | t r| dks|j | | dkr| dk rt t| } nd} tr0|jdkr0t | tjr| |_nDt | trt| |_n*| dkrtd |_ntd | |_n| dk r>| nd|_| |_| |_|jjj}|dk r||kr|jj|||jr|jjjf||d|_| d d|_| d d|_|jjjd krd|_ nP|jj!j"t#krd|_ n4|j$r(|j$%dr(|dk r(||_&d|_ nd|_ |j'f| \|_(|_)|_*t#+}|t,|-|dS)N)config hf_device_mapzThe model has been loaded with `accelerate` and therefore cannot be moved to a specific device. Please discard the `device` argument when creating your pipeline object.tfzDThe AutomaticSpeechRecognitionPipeline is only available in PyTorch.rr'r!cpuzcuda:Z batch_sizeZ num_workersZwhisperseq2seq_whisperseq2seqZWithLM ctc_with_lmctc).rr\rWrSrHr.rUrVgetattr ValueError isinstancerr,nextitervaluesr torchrYstrrZr[task_specific_paramsupdategetZ can_generateZgeneration_configZ call_countpopZ _batch_sizeZ _num_workersZ model_typetype __class____name__rZ_processor_classendswithrT_sanitize_parametersZ_preprocess_paramsZ_forward_paramsZ_postprocess_paramscopyrZcheck_model_type)selfrSr.rHrTrUrVrWrXrYrZr[kwargsr]rlmappingrrr__init__sr         z+AutomaticSpeechRecognitionPipeline.__init__)r-c stj|f|S)a Transcribe the audio sequence(s) given as inputs to text. See the [`AutomaticSpeechRecognitionPipeline`] documentation for more information. Args: inputs (`np.ndarray` or `bytes` or `str` or `dict`): The inputs is either : - `str` that is either the filename of a local audio file, or a public URL address to download the audio file. The file will be read at the correct sampling rate to get the waveform using *ffmpeg*. This requires *ffmpeg* to be installed on the system. - `bytes` it is supposed to be the content of an audio file and is interpreted by *ffmpeg* in the same way. - (`np.ndarray` of shape (n, ) of type `np.float32` or `np.float64`) Raw audio at the correct sampling rate (no further check will be done) - `dict` form can be used to pass raw audio sampled at arbitrary `sampling_rate` and let this pipeline do the resampling. The dict must be in the format `{"sampling_rate": int, "raw": np.array}` with optionally a `"stride": (left: int, right: int)` than can ask the pipeline to treat the first `left` samples and last `right` samples to be ignored in decoding (but used at inference to provide more context to the model). Only use `stride` with CTC models. return_timestamps (*optional*, `str` or `bool`): Only available for pure CTC models (Wav2Vec2, HuBERT, etc) and the Whisper model. Not available for other sequence-to-sequence models. For CTC models, timestamps can take one of two formats: - `"char"`: the pipeline will return timestamps along the text for every character in the text. For instance, if you get `[{"text": "h", "timestamp": (0.5, 0.6)}, {"text": "i", "timestamp": (0.7, 0.9)}]`, then it means the model predicts that the letter "h" was spoken after `0.5` and before `0.6` seconds. - `"word"`: the pipeline will return timestamps along the text for every word in the text. For instance, if you get `[{"text": "hi ", "timestamp": (0.5, 0.9)}, {"text": "there", "timestamp": (1.0, 1.5)}]`, then it means the model predicts that the word "hi" was spoken after `0.5` and before `0.9` seconds. For the Whisper model, timestamps can take one of two formats: - `"word"`: same as above for word-level CTC timestamps. Word-level timestamps are predicted through the *dynamic-time warping (DTW)* algorithm, an approximation to word-level timestamps by inspecting the cross-attention weights. - `True`: the pipeline will return timestamps along the text for *segments* of words in the text. For instance, if you get `[{"text": " Hi there!", "timestamp": (0.5, 1.5)}]`, then it means the model predicts that the segment "Hi there!" was spoken after `0.5` and before `1.5` seconds. Note that a segment of text refers to a sequence of one or more words, rather than individual words as with word-level timestamps. generate_kwargs (`dict`, *optional*): The dictionary of ad-hoc parametrization of `generate_config` to be used for the generation call. For a complete overview of generate, check the [following guide](https://huggingface.co/docs/transformers/en/main_classes/text_generation). max_new_tokens (`int`, *optional*): The maximum numbers of tokens to generate, ignoring the number of tokens in the prompt. Return: `Dict`: A dictionary with the following keys: - **text** (`str`): The recognized text. - **chunks** (*optional(, `List[Dict]`) When using `return_timestamps`, the `chunks` will become a list containing all the various text chunks identified by the model, *e.g.* `[{"text": "hi ", "timestamp": (0.5, 0.9)}, {"text": "there", "timestamp": (1.0, 1.5)}]`. The original full text can roughly be recovered by doing `"".join(chunk["text"] for chunk in output["chunks"])`. )super__call__)rvr-rwrqrrr{&s?z+AutomaticSpeechRecognitionPipeline.__call__c CsLi} |dk r,|jdkr$|s$td|| d<|dk r<|| d<tt} |dk rX|| dd<|dk r|dk rxd|krxtd| d|i} |dk r|| d<|dk r|jdkr|rtd |jd kr|d krtd |jd kr|dkrtd|jdkr |dkr td|| d<|| d<|dk rB|jdkr:td|| d<| | | fS)NraaUsing `chunk_length_s` is very experimental with seq2seq models. The results will not necessarily be entirely accurate and will have caveats. More information: https://github.com/huggingface/transformers/pull/20104. Ignore this warning with pipeline(..., ignore_warning=True)chunk_length_sstride_length_sgenerate_kwargsmax_new_tokenszp`max_new_tokens` is defined both as an argument and inside `generate_kwargs` argument, please use only 1 versiondecoder_kwargszEWe cannot return_timestamps yet on non-CTC models apart from Whisper!rbwordzRCTC with LM can only predict word level timestamps, set `return_timestamps='word'`rc)charrzCTC can either predict character level timestamps, or word level timestamps.Set `return_timestamps='char'` or `return_timestamps='word'` as required.r`rzWhisper cannot return `char` timestamps, only word level or segment level timestamps. Use `return_timestamps='word'` or `return_timestamps=True` respectively.return_timestampsz)Only Whisper can return language for now.return_language)rploggerwarningrdictrerm) rvr}r~Zignore_warningrrrrrZpreprocess_paramsZforward_paramsZpostprocess_paramsrrrrtgsR     z7AutomaticSpeechRecognitionPipeline._sanitize_parametersrc cst|trJ|ds|dr,t|j}nt|d}|}W5QRXt|trbt ||j j }d}i}t|t r| dd}d|krd|ksd|kstd| dd}|dkr| d d| dd}| d}|}|}||j j kr6trd d lm} ntd | t|||j j }|j j |} nd } |dk r|d |d |jd krhtd|jd tt|d | tt|d | f}t|tjstdt|dt|jd krtd|r|dkr|d}t|ttfr||g}t|j j!dd } tt||j j | | } tt|d |j j | | } tt|d |j j | | }| | |krtd|jdk}t"||j | | |||j#D] }|Vqnd|j ||j j dd}|j#dk r|j$|j#d}|dk r |jdkrtd||d<ddi||VdS)Nzhttp://zhttps://rbrr#rawr=zWhen passing a dictionary to AutomaticSpeechRecognitionPipeline, the dict needs to contain a "raw" key containing the numpy array representing the audio and a "sampling_rate" key, containing the sampling_rate associated with that arraypathr) functionalztorchaudio is required to resample audio samples in AutomaticSpeechRecognitionPipeline. The torchaudio package can be installed through: `pip install torchaudio`.r zStride is too large for inputz)We expect a numpy ndarray as input, got ``zMWe expect a single channel audio input for AutomaticSpeechRecognitionPipelineinputs_to_logits_ratioz.Chunk length must be superior to stride lengthr`r!r"r$raz8Stride is only usable with CTC models, try removing it !r)T)%rfrk startswithrequestsrncontentopenreadbytesrr.r#rrorer Z torchaudior ImportErrorZresamplerjZ from_numpynumpyr*rrr<ndarrayrpr;floatrdrSr\r6rZr,)rvr-r}r~frextraZ_inputsZin_sampling_rateFrZalign_tor/r0r1r2itemr5rrr preprocesss          0        z-AutomaticSpeechRecognitionPipeline.preprocesscCs|dkr i}|dd}|dd}|d}|jdkr2|j}d|krX|d}n&d|krl|d}ntd||r|jdkr||d <|d krd |d <|dk r|d |jj|d<|jjf|||d|d|} |d kr |jdkr | d| dd} nd| i} |jdkr|dk r|| d<n|d} |j| |d} | j } |jdkrfd| i} nd| j ddi} |dk rd|jj j }t |trt|g|d | d<nt||| d<|}d|i| |S)Nattention_maskrr)rar`r&r(zhSeq2Seq speech recognition model requires either a `input_features` or `input_values` key, but only has r`rrTZreturn_token_timestampsrZ num_frames)r)Zencoder_outputsrrLtoken_timestamps)tokensrr)r(rrblogitsr')dimr )rorprSZ get_encoderrekeysr.Z hop_lengthgeneraterZargmaxr\rrftupler )rvZ model_inputsrrrrr)encoderr-routr(outputsrrrrrr_forward sZ                 z+AutomaticSpeechRecognitionPipeline._forward)rc$Cs i}g}|jdkrdnd}d}|D]^} | |} | dd}|dk rv|jdkrv|\} } } | | }| dd| |f} || q"|r|jdkrt||j} n|jdkr$|jj|jj j }|jj }|D]@}d|kr|d\}}}||}||}||}|||f|d<q|jj ||||d\}}nt j|d d } | d } |jdkr|dkrVi}|jj| f|}|d d }|r|d d }g}|D] \}\}}||||d qn^|jdkr|jdk}|jj| |d}|r|jj| |ddd}|dkr|j||jj}|r|jdkrg}|D]Z}|d|jj j}||jj }|d|jj j} | |jj } ||||| fdq(||d<tt}!|D]d}|dd|dd|dd|dd|dd|D]\}"}#|!|"|#qqd|i||!S)Nrbrrr>rbrcrar`)rrtime_precisionr )Zaxisrr)r start_offset end_offsetrc)skip_special_tokensT)rZoutput_char_offsetsZ char_offsetsrrrr)text timestampchunksr)rr)rprrnrrPrHr. chunk_lengthrSr\max_source_positionsr#Z _decode_asrr<Z concatenatesqueezerTZ decode_beamsdecodeZ_get_word_offsetsZreplace_word_delimiter_charrrlistroitems)$rvZ model_outputsrrroptionalZ final_itemskeyrrrZtotal_nrrZright_nrr#outputr/r0r1rZbeamsZ chunk_offsetoffsetsrrrrrrstartstoprkvrrr postprocessMs                     z.AutomaticSpeechRecognitionPipeline.postprocess) NNNNNrRNNNF)NNNNNNNN)rN)FN)NNN)rr __module__ __qualname____doc__rrkrr rrrboolryr<rrr{rtrrrr __classcell__rrr|rrQsVC   `C @ d CrQc(CsP|dd}g}|j|}d}t|D]\}} | \} } t| trNt| } | \} } }| d} || krt| |kddnd}| |d} | |k}|dkr*t |dkr*t|dd|dd@dd}t|dd}||krt ||n|}|| |8}t ||j |}t | |j |}t| |||kd}|j ddkr*|ddkrx||ddn|d}d}g}tt|D]P\}}|dd}|d|||kr|dkrqt|dkrt| d||\}}}|dkr||kr|}|}t| |dd|kddd}|d|}|dkr|t|krt| |d|d|d}|d|d<n\|dkr| |d|}|dkr|d|dn|dg} t|d| }|d|7<qt|dkr*||t||d<|dt||}| |d} | |k}t|dd|dd@dd}t |dkrt|dd}||krt ||dn|}t|dkrd}!|D]l}"|dks|!dkr|ddn| d}#| |!|"}$|$d|$d}%|#|$d<|#|%|$d<| |$|"}!q|| 7}q(g}&tt|D]}'|&||'7}&q4|&S)a Computes the final sequences by merging the end of the nth sequence with the beginning of the n+1th sequence. Since `WhisperForConditionalGeneration` produces the timestamps pairwise, we filter the consecutive timestamps and only iterate over them. We keep track of the `time` which indicates the actual starting time of the chunk that is processed. We need to make sure to offset the timestamps tokens by the `time` in order for the tokenizer to properly compute the final `offset`. z<|notimestamps|>r rNr')Zconvert_tokens_to_idsr enumeraterfrr<r=rwhererJrrr#r*reversedr;rDinsertr+rI)(rLrHr.rZtimestamp_beginrrtimeZseq_idxrrMrr/r0r1Z begin_idxZtimestamp_tokensZ consecutiveZlast_timestampoffsetZ overlap_timeZrelevant_timestamp best_matchZsliced_sequenceidxZprevious_sequenceZprevious_tokensrBrCZ match_lengthZbest_idxZend_of_curr_sequence_idxZprevious_sliceZ last_sliceZ current_sliceZ actual_offsetZ sliced_tokensdurationresultr@rrr_find_timestamp_sequences      " & $    $  " &(    r)TN)+ collectionsrtypingrrrrrr<rrUrZtokenization_utilsr utilsr r r Z audio_utilsrbaserrrZ pyctcdecoderZ!feature_extraction_sequence_utilsrZmodeling_utilsrZ get_loggerrrrrjZmodels.auto.modeling_autorrr r6rDrPrQrrrrrs4         '