HarmonyOS TextToSpeechEngine接收到音频流如何通过audiorenderer播放,请提供一个demo

HarmonyOS
1天前
浏览
收藏 0
回答 1
待解决
回答 1
按赞同
/
按时间
Heiang

示例代码如下:

1、TtsHelper文件

import { textToSpeech } from '@kit.CoreSpeechKit';
import { BusinessError } from '@kit.BasicServicesKit';
import { audio } from '@kit.AudioKit';
import { HashMap, JSON, TreeMap } from '@kit.ArkTS';

export { TtsHelper, TtsHelperCallback }

const TAG: string = "[TtsHelper]";

class TtsHelper {
  /**
   * tts引擎
   */
  private mTtsEngine: textToSpeech.TextToSpeechEngine | undefined = undefined;
  /**
   * 创建音频渲染
   */
  private mAudioRenderer: audio.AudioRenderer | undefined = undefined;
  /**
   * 回调
   */
  private mTtsHelperCallback: TtsHelperCallback | undefined = undefined;
  /**
   * 合成后的音频 PCM 数据
   */
  private mAudioPcmDataMap: HashMap<string, AudioPcmDataOperate> = new HashMap();
  /**
   * 合成时,传入的合成播报ID
   */
  private mRequestId: string | undefined;
  /**
   * speak的回调信息
   */
  private mSpeakListener: textToSpeech.SpeakListener = {
    /**
     * 开始播报回调
     */
    onStart: async (requestId: string, startResponse: textToSpeech.StartResponse): Promise<void> => {
      console.info(TAG, `onStart, requestId: ${requestId} response: ${JSON.stringify(startResponse)}`);
      this.mRequestId = requestId;
      // TODO 此处调用会报错:Code: 6800301, message: system error.
      // 初始化音频渲染,startResponse:返回播放的相关参数
      // await this.initAudioRendererByStartResponse(startResponse);
    },
    /**
     * 合成完成及播报完成回调
     */
    onComplete: (requestId: string, response: textToSpeech.CompleteResponse): void => {
      console.info(TAG, `onComplete, requestId: ${requestId} response: ${JSON.stringify(response)}`);
    },
    /**
     * 停止播报回调
     */
    onStop: (requestId: string, response: textToSpeech.StopResponse): void => {
      this.mRequestId = undefined;
      console.info(TAG, `onStop, requestId: ${requestId} response: ${JSON.stringify(response)}`);
    },
    /**
     * 返回音频流
     */
    onData: async (requestId: string, audio: ArrayBuffer, response: textToSpeech.SynthesisResponse): Promise<void> => {
      console.info(TAG,
        `onData, requestId: ${requestId} sequence: ${JSON.stringify(response)} audioLength: ${audio.byteLength}`);
      await this.handleAudioPcmData(requestId, audio, response);
    },
    /**
     * 错误回调
     */
    onError: (requestId: string, errorCode: number, errorMessage: string): void => {
      console.error(TAG, `onError, requestId: ${requestId} errorCode: ${errorCode} errorMessage: ${errorMessage}`);
    }
  }

  /**
   * 处理合成后的音频 PCM 数据
   * @param requestId 合成播报ID,全局不允许重复。
   * @param audio pcm数据
   * @param response 响应内容
   * buffer:https://developer.huawei.com/consumer/cn/doc/atomic-references/js-apis-buffer-0000001824697897
   */
  private async handleAudioPcmData(requestId: string, audio: ArrayBuffer, response: textToSpeech.SynthesisResponse) {
    if (!this.mAudioPcmDataMap) {
      return
    }
    // 存储音频数据
    let audioPcmData = this.mAudioPcmDataMap.get(requestId);
    if (!audioPcmData) {
      audioPcmData = new AudioPcmDataOperate(requestId);
      audioPcmData.putValue(response.sequence, audio, response);

      this.mAudioPcmDataMap.set(requestId, audioPcmData);
      return
    }
    audioPcmData.putValue(response.sequence, audio, response);
    this.mAudioPcmDataMap.replace(requestId, audioPcmData);
  }

  /**
   * 音频渲染器写入数据回调,
   */
  private mAudioRendererWriteDataCallback = async (buf: ArrayBuffer) => {
    console.info(TAG, "mAudioRendererWriteDataCallback, bufLength: " + buf.byteLength);

    // 音频数据为空,则不处理,
    let audioPcmData: AudioPcmDataOperate | undefined = this.getAudioPcmData(this.mRequestId);
    if (audioPcmData === undefined) {
      return
    }
    // bufLength: 2972
    // 播放正常:将音频流数据保存到同一个Uint8Array中,然后从Uint8Array中截取渲染器buf长度的音频流,填充到渲染器buf中,可以正常播放;
    audioPcmData.writeAudioBufferByCompletePcmData(buf);
  }

  /**
   * 根据requestId,获取AudioPcmData数据
   * @param requestId
   * @returns
   */
  private getAudioPcmData(requestId?: string): AudioPcmDataOperate | undefined {
    // 音频数据为空,则不处理
    if (!requestId || !this.mAudioPcmDataMap || this.mAudioPcmDataMap.isEmpty()) {
      return undefined;
    }
    let audioPcmData: AudioPcmDataOperate = this.mAudioPcmDataMap.get(this.mRequestId);
    if (audioPcmData) {
      return audioPcmData;
    }
    return undefined;
  }

  /**
   * 初始化
   */
  public async initHelper(callback?: TtsHelperCallback) {
    try {
      this.mTtsHelperCallback = callback;
      if (!canIUse('SystemCapability.AI.TextToSpeech')) {
        this.printErrMsg('', -1, `initHelper failed. The device does not support TextToSpeech.`);
        return
      }

      if (!this.mTtsEngine) {
        // 创建Tts引擎,当前仅支持单应用、单实例.
        this.mTtsEngine = await this.createTtsEngine();
        // 设置回调
        this.mTtsEngine.setListener(this.mSpeakListener);
      }

      if (!this.mAudioRenderer) {
        this.mAudioRenderer = await this.createAudioRenderer();
        this.mAudioRenderer.on('writeData', this.mAudioRendererWriteDataCallback);
      }
    } catch (e) {
      const err = e as BusinessError;
      // 创建引擎失败时返回错误码1003400005,可能原因:引擎不存在、资源不存在、创建引擎超时
      console.error(TAG, `initHelper failed. Code: ${err.code}, message: ${err.message}.`);
      this.printErrMsg('', -1, `initHelper failed. Code: ${err.code}, message: ${err.message}.`);
    }
  }

  /**
   * 开始
   * @param requestId
   * @param originalText
   */
  public async start(requestId: string, originalText: string) {
    await this.startSynthesisVoice(requestId, originalText);
    await this.startAudioRenderer();
  }

  /**
   * 暂停
   */
  public async pause() {
    await this.pauseAudioRenderer();
  }

  /**
   * 停止
   */
  public async stop() {
    await this.stopAudioRenderer();
  }

  /**
   * 创建Tts引擎
   * @param engineName 引擎名称。 当前仅支持单应用、单实例。
   * @returns
   */
  private async createTtsEngine(engineName: string = 'EngineName'): Promise<textToSpeech.TextToSpeechEngine> {
    console.log(TAG, 'createTtsEngine.');
    // 当前仅支持单应用、单实例
    // 设置创建引擎参数
    let extraParam: Record<string, Object> = { "style": 'interaction-broadcast', "locate": 'CN', "name": engineName };
    let initParamsInfo: textToSpeech.CreateEngineParams = {
      language: 'zh-CN',
      // 模式。 0为在线,目前不支持;1为离线,当前仅支持离线模式。
      online: 1,
      // 音色。 0为聆小珊女声音色,当前仅支持聆小珊女声音色。
      person: 0,
      extraParams: extraParam
    };
    return await textToSpeech.createEngine(initParamsInfo);
  }

  /**
   * 合成语音,返回音频流
   * @param requestId 合成播报ID,全局不允许重复。
   * @param originalText 待合成播报的文本。
   */
  public async startSynthesisVoice(requestId: string, originalText: string) {
    if (!this.mTtsEngine) {
      this.printNotInitErr();
      return
    }
    // 判断当前是否处于合成或播报状态,可以通过listener方法接收isBusy错误相关的回调
    if (this.mTtsEngine.isBusy()) {
      this.mTtsEngine.stop();
    }
    // 设置播报相关参数
    let extraParam: Record<string, Object> = {
      // 语速:可选,支持范围[0.5-2],不传参时默认为1。
      "speed": 1,
      // 音量: 可选,支持范围[0-2],不传参时默认为1。
      "volume": 2,
      // 音调: 可选,支持范围[0.5-2],不传参时默认为1
      "pitch": 1,
      // 通道: 可选,参数范围0-16,整数类型; 不传参时默认为3,语音助手通道。
      "soundChannel": 3,
      // 播报模式: 可选,不传参时默认为0; 0:排队模式播报; 1:抢占模式播报。
      "queueMode": 0,
      // 语境: 播放阿拉伯数字用的语种; 可选,当前仅支持“zh-CN”中文,不传参时默认“zh-CN”。
      "languageContext": 'zh-CN',
      // 音频类型: 可选,当前仅支持“pcm”,不传参时默认为“pcm”(PCM 即脉冲编码调制 (Pulse Code Modulation))。
      "audioType": "pcm",
      // 合成类型: 可选,不传参时默认为1; 0:仅合成不播报,返回音频流; 1:合成与播报不返回音频流。
      "playType": 0,
      // 支持的功能:返回持续时间信息
      "supportFeatures": "{\"supportDurationInfo\": true}"
    };

    /**
     * 合成播报音频流的相关参数,用于配置语速、音量、音调、合成类型等。
     */
    let speakParams: textToSpeech.SpeakParams = {
      // requestId在同一实例内仅能用一次,请勿重复设置
      requestId: requestId,
      extraParams: extraParam
    };
    // 调用播报方法
    this.mTtsEngine.speak(originalText, speakParams);
  }

  /**
   * 创建音频渲染
   * @param sampleRate 采样率信息
   * @param sampleBit 采样位数
   * @param audioChannel 通道数信息
   * @returns
   */
  private async createAudioRenderer(sampleRate: number = audio.AudioSamplingRate.SAMPLE_RATE_16000,
    sampleBit: number = audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE,
    audioChannel: number = audio.AudioChannel.CHANNEL_1): Promise<audio.AudioRenderer> {
    console.log(TAG, 'createAudioRenderer.');
    // 音频流信息
    let audioStreamInfo: audio.AudioStreamInfo = {
      // 音频文件的采样率
      samplingRate: sampleRate,
      // 音频文件的通道数
      channels: audioChannel,
      // 音频采样格式
      sampleFormat: sampleBit,
      // 音频编码格式
      encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW
    };
    // 渲染器信息
    let audioRendererInfo: audio.AudioRendererInfo = {
      // 音频流使用类型
      usage: audio.StreamUsage.STREAM_USAGE_VOICE_ASSISTANT,
      // 音频渲染器标志:0代表普通音频渲染器,1代表低时延音频渲染器。ArkTS接口暂不支持低时延音频渲染器。
      rendererFlags: 0
    };
    // 音频渲染器选项信息
    let audioRendererOptions: audio.AudioRendererOptions = {
      streamInfo: audioStreamInfo,
      rendererInfo: audioRendererInfo
    };
    // 创建AudioRenderer实例
    return await audio.createAudioRenderer(audioRendererOptions);
  }

  /**
   * 开始音频渲染
   */
  public async startAudioRenderer() {
    console.log(TAG, "startAudioRenderer.");
    if (!this.mAudioRenderer) {
      this.printNotInitErr();
      return
    }
    console.log(TAG, "startAudioRenderer. state: " + this.mAudioRenderer.state);
    // 当且仅当状态为prepared、paused和stopped之一时才能启动渲染
    let stateGroup = [audio.AudioState.STATE_PREPARED, audio.AudioState.STATE_PAUSED, audio.AudioState.STATE_STOPPED];
    if (stateGroup.indexOf((this.mAudioRenderer as audio.AudioRenderer).state.valueOf()) === -1) {
      console.error(TAG, 'startAudioRenderer failed');
      return;
    }
    // 启动渲染
    this.mAudioRenderer.start((err: BusinessError) => {
      if (err) {
        console.error(TAG, 'startAudioRenderer, Renderer start failed.');
      } else {
        console.info(TAG, 'startAudioRenderer, Renderer start success.');
      }
    });
  }

  /**
   * 暂停渲染
   */
  public async pauseAudioRenderer() {
    console.log(TAG, "pauseAudioRenderer.");
    if (!this.mAudioRenderer) {
      this.printNotInitErr();
      return
    }
    console.log(TAG, "pauseAudioRenderer. state: " + this.mAudioRenderer.state);
    // 只有渲染器状态为running的时候才能暂停
    if (this.mAudioRenderer.state.valueOf() !== audio.AudioState.STATE_RUNNING) {
      console.info(TAG, 'pauseAudioRenderer, Renderer is not running.');
      return;
    }
    // 暂停渲染
    this.mAudioRenderer.pause((err: BusinessError) => {
      if (err) {
        console.error(TAG, 'pauseAudioRenderer, Renderer pause failed.');
      } else {
        console.info(TAG, 'pauseAudioRenderer, Renderer pause success.');
      }
    });
  }

  /**
   * 停止渲染
   */
  public async stopAudioRenderer() {
    console.log(TAG, "stopAudioRenderer.");
    if (!this.mAudioRenderer) {
      this.printNotInitErr();
      return
    }
    console.log(TAG, "stopAudioRenderer. state: " + this.mAudioRenderer.state);
    // 只有渲染器状态为running或paused的时候才可以停止
    if (this.mAudioRenderer.state.valueOf() !== audio.AudioState.STATE_RUNNING
      && this.mAudioRenderer.state.valueOf() !== audio.AudioState.STATE_PAUSED) {
      console.info(TAG, 'stopAudioRenderer, Renderer is not running or paused.');
      return;
    }
    // 停止渲染
    this.mAudioRenderer.stop((err: BusinessError) => {
      if (err) {
        console.error(TAG, 'stopAudioRenderer, Renderer stop failed.');
      } else {
        // 触发停止时,重置状态值
        let audioPcmData: AudioPcmDataOperate | undefined = this.getAudioPcmData(this.mRequestId);
        if (audioPcmData) {
          audioPcmData.resetStateValue();
        }
        console.info(TAG, 'stopAudioRenderer, Renderer stop success.');
      }
    });
  }

  /**
   * 释放
   */
  public release() {
    console.log(TAG, "release.");
    if (this.mTtsEngine) {
      this.mTtsEngine.stop();
      this.mTtsEngine.shutdown();
      this.mTtsEngine = undefined;
      console.log(TAG, "release, mTtsEngine shutdown.");
    }
    // 释放音频渲染
    if (this.mAudioRenderer) {
      // 渲染器状态不是released状态,才能release
      if (this.mAudioRenderer.state.valueOf() === audio.AudioState.STATE_RELEASED) {
        console.info(TAG, 'release, Renderer already released.');
        return;
      }
      this.mAudioRenderer.release((err: BusinessError) => {
        if (err) {
          console.error(TAG, 'release, Renderer release failed.');
        } else {
          console.info(TAG, 'release, Renderer release success.');
        }
      });
      this.mAudioRenderer = undefined;
    }
  }

  private printErrMsg(requestId: string, errorCode: number, errorMessage: string) {
    console.error(TAG, `printlnErr, requestId:${requestId}, errorCode: ${errorCode}, errorMessage: ${errorMessage}`);
    if (this.mTtsHelperCallback) {
      this.mTtsHelperCallback.onError(requestId, errorCode, errorMessage);
    }
  }

  private printNotInitErr() {
    console.error(TAG, "TtsHelper not init.");
    if (this.mTtsHelperCallback) {
      this.mTtsHelperCallback.onError('', -1, 'TtsHelper not init.');
    }
  }
}

/**
 * tts助手回调
 */
interface TtsHelperCallback {
  /**
   * 错误回调
   * @param requestId
   * @param errorCode
   * @param errorMessage
   */
  onError: (requestId: string, errorCode: number, errorMessage: string) => void;
}

/**
 * 存储音频PCM数据
 */
class AudioPcmDataOperate {
  /**
   * 合成时的请求ID
   */
  private _requestId: string;
  /**
   * 使用TreeMap接收每一条pcmData数据
   */
  private _pcmDataMap: TreeMap<number, Uint8Array> = new TreeMap();
  /**
   * 使用TreeMap接收每一条pcmData数据对应的 合成文本信息
   */
  private _synthesisMap: TreeMap<number, textToSpeech.SynthesisResponse> = new TreeMap();
  /**
   * 保存完整的音频数据
   */
  private mCompletePcmData: Uint8Array | undefined = undefined;
  /**
   * 完整音频数据的开始下标:由于合成的pcmData数据并非是有序返回的,需要记录下标,根据下标刷新 mCompletePcmData
   */
  private mCompletePcmDataIndex: number = 0;
  /**
   * 完整音频数据 当前偏移量:在向渲染器写入音频时,记录当前已写的大小
   */
  private mCompletePcmDataOffset: number = 0;

  constructor(requestId: string) {
    this._requestId = requestId;
  }

  public set requestId(value: string) {
    this._requestId = value;
  }

  public get requestId(): string {
    return this._requestId;
  }

  public set pcmDataMap(value: TreeMap<number, Uint8Array>) {
    this._pcmDataMap = value;
  }

  public get pcmDataMap(): TreeMap<number, Uint8Array> {
    return this._pcmDataMap;
  }

  public set synthesisMap(value: TreeMap<number, textToSpeech.SynthesisResponse>) {
    this._synthesisMap = value;
  }

  public get synthesisMap(): TreeMap<number, textToSpeech.SynthesisResponse> {
    return this._synthesisMap;
  }

  /**
   * 设置数据,并刷新完整数据,使用TreeMap存储,刷新完整数据时,从TreeMap中取数据进行刷新;以防止sequenceKey出现乱序的情况。
   * @param sequenceKey
   * @param audioBuffer
   * @param ttsSynthesisInfo
   */
  public putValue(sequenceKey: number, audioBuffer: ArrayBuffer, ttsSynthesisInfo: textToSpeech.SynthesisResponse) {
    this.pcmDataMap.set(sequenceKey, new Uint8Array(audioBuffer));
    this.synthesisMap.set(sequenceKey, ttsSynthesisInfo);
    // 刷新完整的PCM数据
    this.refreshCompletePcmData();
  }

  /**
   * 刷新完整数据
   */
  public refreshCompletePcmData() {
    for (let index = this.mCompletePcmDataIndex; index < this.pcmDataMap.length; index++) {
      const pcmDataUint8Array = this.pcmDataMap.get(index);
      // 有值,刷新数据;这个值可以写到文件
      if (pcmDataUint8Array) {
        if (this.mCompletePcmData === undefined) {
          this.mCompletePcmData = new Uint8Array(pcmDataUint8Array);
        } else {
          const tempLength = this.mCompletePcmData.byteLength + pcmDataUint8Array.byteLength;
          const temp = new Uint8Array(tempLength);
          temp.set(this.mCompletePcmData);
          temp.set(pcmDataUint8Array, this.mCompletePcmData.byteLength);
          this.mCompletePcmData = temp;
        }
        this.mCompletePcmDataIndex++;
      } else {
        this.mCompletePcmDataIndex = index;
        return
      }
    }
  }

  /**
   * 向音频渲染器写音频流数据
   * @param buf 音频渲染器返回的buf
   */
  public writeAudioBufferByCompletePcmData(buf: ArrayBuffer) {
    //存储tts合成的音频信息;
    if (this.mCompletePcmData === undefined) {
      return
    }
    if (this.mCompletePcmDataOffset >= this.mCompletePcmData.byteLength) {
      return
    }
    const bufUint8 = new Uint8Array(buf);

    let offset = bufUint8.byteLength + this.mCompletePcmDataOffset;
    if (offset >= this.mCompletePcmData.byteLength) {
      bufUint8.set(this.mCompletePcmData.subarray(this.mCompletePcmDataOffset));
    } else {
      bufUint8.set(this.mCompletePcmData.subarray(this.mCompletePcmDataOffset, offset));
    }
    this.mCompletePcmDataOffset += bufUint8.byteLength;
  }

  /**
   * 重置状态值
   */
  public resetStateValue() {
    this.mCompletePcmDataOffset = 0;
  }
}

2、page文件

import { textToSpeech } from '@kit.CoreSpeechKit';
import { TtsHelper } from '../manager/ttsHelper';

let ttsEngine: textToSpeech.TextToSpeechEngine;

@Entry
@Component
struct myTextToSpeechEngine0708page {
  @State createCount: number = 0;
  @State requestId: number = 0;
  @State ttsHelper: TtsHelper | undefined = undefined
  @State status: boolean = false
  @State originalText: string = "xxxxx"

  build() {
    Column() {
      Scroll() {
        Column() {
          TextArea({ placeholder: 'Please enter tts original text', text: `${this.originalText}` })
            .margin(20)
            .focusable(false)
            .border({
              width: 5,
              color: 0x317AE7,
              radius: 10,
              style: BorderStyle.Dotted
            })
            .onChange((value: string) => {
              this.originalText = value;
              console.info("original text: " + this.originalText);
            })

          Button() {
            Text("初始化")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.createCount++;
            console.info(`CreateTtsEngine:createCount:${this.createCount}`);
            this.ttsHelper = new TtsHelper();
            this.ttsHelper.initHelper();
          })

          Button() {
            Text("开始")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            if (this.ttsHelper) {
              if (this.status) {
                // requestId传之前的会接着暂停的地方开始
                this.ttsHelper.start(this.requestId.toString(), this.originalText)
                this.status = false;
                console.log('requestID pause : ' + this.requestId)
              } else {
                this.ttsHelper.start((++this.requestId).toString(), this.originalText)
                console.log('requestID start : ' + this.requestId)
              }
            }
          })
          Button() {
            Text("暂停")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            if (this.ttsHelper) {
              this.ttsHelper.pause();
              this.status = true;
            }
          })
          Button() {
            Text("停止")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            if (this.ttsHelper) {
              this.ttsHelper.stop()
            }
          })

          Button() {
            Text("释放")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            if (this.ttsHelper) {
              this.ttsHelper.release()
            }
          })
        }
        .layoutWeight(1)
      }
      .width('100%')
      .height('100%')
    }
  }
}
分享
微博
QQ
微信
回复
1天前
相关问题
请提供HarmonyOS音频播放器实例代码
451浏览 • 0回复 待解决
HarmonyOS 请提供路由跳转Demo
501浏览 • 1回复 待解决
使用AudioRenderer播放pcm音频流失败
1855浏览 • 1回复 待解决
HarmonyOS 请提供音频编码示例
28浏览 • 1回复 待解决
AudioCapturer录音+AudioRenderer播放音频
1547浏览 • 1回复 待解决
HarmonyOS 请提供AVRecorder demo示例
187浏览 • 1回复 待解决
HarmonyOS能否提供一个NFC识别的demo
543浏览 • 1回复 待解决
使用AudioRenderer开发音频播放功能
1216浏览 • 1回复 待解决
HarmonyOS 如何实现下列功能,请提供demo
627浏览 • 1回复 待解决
HarmonyOS 如何播放rtmp协议的音频
16浏览 • 1回复 待解决
HarmonyOS 能否提供一个视频压缩的demo
245浏览 • 1回复 待解决
请提供HarmonyOS硬编硬解demo
640浏览 • 1回复 待解决