/*
* Copyright 2024 KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 3 of the License, or (at your option) any later
* version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see .
*/
#include "xunfeispeechengine_p.h"
#include
#include
#include
#include "xunfei/logger.h"
#include "servererror.h"
#include "util.h"
XunfeiSpeechEnginePrivate::XunfeiSpeechEnginePrivate() {
hlog_set_level(LOG_LEVEL_SILENT);
}
XunfeiSpeechEnginePrivate::~XunfeiSpeechEnginePrivate() {}
std::string XunfeiSpeechEnginePrivate::engineName() const { return "xunfei"; }
std::string XunfeiSpeechEnginePrivate::currentModel() const {
return currentModel_;
}
std::string XunfeiSpeechEnginePrivate::currentModelName() const {
return currentModelName_;
}
void XunfeiSpeechEnginePrivate::setConfig(const std::string &config) {
Json::Value configJson = xunfei_speech_util::formatJsonFromString(config);
if (!configJson.isNull() && configJson.isMember("apiKeyRealTime") &&
configJson.isMember("appIdRealTime") &&
configJson["appIdRealTime"].isString() &&
configJson["apiKeyRealTime"].isString()) {
appIdRealTime_ = configJson["appIdRealTime"].asString();
apiKeyRealTime_ = configJson["apiKeyRealTime"].asString();
} else {
LOGE("Invalid config for xunfei realtime asr engine:", config, ".");
}
if (!configJson.isNull() && configJson.isMember("apiKeyContinuous") &&
configJson.isMember("secretKeyContinuous") &&
configJson.isMember("appIdContinuous") &&
configJson["apiKeyContinuous"].isString() &&
configJson["secretKeyContinuous"].isString() &&
configJson["appIdContinuous"].isString()) {
appIdContinuous_ = configJson["appIdContinuous"].asString();
apiKeyContinuous_ = configJson["apiKeyContinuous"].asString();
secretKeyContinuous_ = configJson["secretKeyContinuous"].asString();
} else {
LOGE("Invalid config for xunfei iat&tts engine:", config, ".");
}
}
bool XunfeiSpeechEnginePrivate::setCurrentModel(const std::string &model) {
Json::Value modelInfoJson =
xunfei_speech_util::formatJsonFromString(modelInfo());
for (auto &item : modelInfoJson["models"]) {
if (item["model"].asString() == model) {
currentModel_ = model;
return true;
}
}
LOGE("Invalid current model name:", model, ".");
return false;
}
std::string XunfeiSpeechEnginePrivate::modelInfo() const {
const char *info = R"(
{
"vendor": "讯飞",
"models": [
{
"name": "讯飞-语音大模型",
"model": "讯飞-语音大模型",
"subConfig": [
{
"name": "语音听写&语音合成",
"config": [
{
"friendlyName": "APPID",
"configName": "appIdContinuous"
},
{
"friendlyName": "APIKey",
"configName": "apiKeyContinuous"
},
{
"friendlyName": "APISecret",
"configName": "secretKeyContinuous"
}
]
},
{
"name": "实时语音转写",
"config": [
{
"friendlyName": "APPID",
"configName": "appIdRealTime"
},
{
"friendlyName": "APIKey",
"configName": "apiKeyRealTime"
}
]
}
],
"is3rdModel": true
}
]
}
)";
return info;
}
void XunfeiSpeechEnginePrivate::setRecognitionResultCallback(
ai_engine::core_ai::speech::RecognitionResultCallback callback) {
recognitionCallback_ = callback;
}
void XunfeiSpeechEnginePrivate::setSynthesisResultCallback(
ai_engine::core_ai::speech::SynthesisResultCallback callback) {
synthesisCallback_ = callback;
}
void XunfeiSpeechEnginePrivate::setRecognitionParams(
const std::string ¶ms) {
Json::Value paramsJson = xunfei_speech_util::formatJsonFromString(params);
format_ = paramsJson["format"].asString();
rate_ = paramsJson["rate"].asInt();
channel_ = paramsJson["channel"].asInt();
if (format_.empty()) {
format_ = "pcm";
}
if (rate_ == 0) {
rate_ = 16000;
}
if (channel_ == 0) {
channel_ = 1;
}
}
void XunfeiSpeechEnginePrivate::setSynthesisParams(const std::string ¶ms) {
Json::Value paramsJson = xunfei_speech_util::formatJsonFromString(params);
if (paramsJson.isMember("speed") && paramsJson["speed"].isInt()) {
speed_ = paramsJson["speed"].asInt();
}
if (paramsJson.isMember("volume") && paramsJson["volume"].isInt()) {
volume_ = paramsJson["volume"].asInt();
}
if (paramsJson.isMember("pitch") && paramsJson["pitch"].isInt()) {
pitch_ = paramsJson["pitch"].asInt();
}
if (speed_ < 0 || speed_ > 100) {
speed_ = 50;
}
if (volume_ < 0 || volume_ > 100) {
volume_ = 50;
}
if (pitch_ < 0 || pitch_ > 100) {
pitch_ = 50;
}
}
std::string XunfeiSpeechEnginePrivate::continuousAsrBuildUrl() {
const std::string ts = xunfei_speech_util::getCurrentUnixTimestamp();
if (ts.empty()) {
return std::string{};
}
std::string signInput = appIdRealTime_ + ts;
signInput = xunfei_speech_util::calculateMD5(signInput);
signInput =
xunfei_speech_util::hmacSha1Encode(signInput, apiKeyRealTime_);
return std::string(realtimeAsrHostUrl_) + "?" + "appid=" + appIdRealTime_ +
"&ts=" + xunfei_speech_util::urlEncode(ts) +
"&signa=" + xunfei_speech_util::urlEncode(signInput) + "&roleType=2";
}
void XunfeiSpeechEnginePrivate::handleRecognitionResult(
const Json::Value &jsonData) {
RecognitionResultList resultList = getContinuousRecognitionResult(jsonData);
if (recognitionCallback_) {
for (const auto &result : resultList) {
recognitionCallback_(result);
}
}
}
bool XunfeiSpeechEnginePrivate::isRecognizingResult(
const Json::Value &jsonData) {
return jsonData["cn"]["st"]["type"].asString() == "1";
}
XunfeiSpeechEnginePrivate::RecognitionResultList
XunfeiSpeechEnginePrivate::getContinuousRecognitionResult(
const Json::Value &jsonData) {
RecognitionResultList resultList;
for (int k = 0; k < jsonData["cn"]["st"]["rt"].size(); k++) {
for (int i = 0; i < jsonData["cn"]["st"]["rt"][k]["ws"].size(); i++) {
for (int j = 0;
j < jsonData["cn"]["st"]["rt"][k]["ws"][i]["cw"].size(); j++) {
const std::string resultText =
jsonData["cn"]["st"]["rt"][k]["ws"][i]["cw"][j]["w"]
.asString();
// 该值只有角色切换时才会变化,其余时值为0。例如角色A开始说话
// rl=1,后面角色A说话rl都是0,等到角色B开始说话时,rl=2,角色
// B继续说话rl又变回0。
const std::string roleText =
jsonData["cn"]["st"]["rt"][k]["ws"][i]["cw"][j]["rl"]
.asString();
ai_engine::core_ai::speech::RecognitionResult result{};
result.success = true;
result.error = EngineError{};
result.modelName = currentModelName_;
result.text = resultText;
result.speakerId = 1;
if (isRecognizingResult(jsonData)) {
result.reason = ai_engine::core_ai::speech::
SpeechResultReason::SpeechRecognizing;
} else {
result.reason = ai_engine::core_ai::speech::
SpeechResultReason::SpeechRecognized;
}
int role = atoi(roleText.c_str());
if (role != currentRole_ && role != 0) {
result.speakerId = role;
currentRole_ = role;
resultList.push_back(result);
continue;
}
if (resultList.empty()) {
resultList.push_back(result);
} else {
resultList.back().text += resultText;
}
}
}
}
return resultList;
}
void XunfeiSpeechEnginePrivate::handleMessage(const std::string &message) {
LOGI("Continuous asr: on message", message);
if (message.empty()) {
return;
}
int code = xunfei_speech_server_error::parseErrorCode(message);
if (code == -1) {
// 回复消息体不完整,不是json结构
ai_engine::core_ai::speech::RecognitionResult result{};
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Continuous asr: on message: message is incomplete.");
result.modelName = currentModelName_;
result.text = "";
result.speakerId = -1;
result.reason =
ai_engine::core_ai::speech::SpeechResultReason::SpeechErrorOccurred;
if (recognitionCallback_) {
recognitionCallback_(result);
}
} else if (code == 0) {
Json::Value resultJson = xunfei_speech_util::stringToJson(message);
if (resultJson.isMember("action") && resultJson["action"].isString()) {
std::string action = resultJson["action"].asString();
if (action == "started") {
LOGI("started");
} else if (action == "result") {
Json::Value dataJson = xunfei_speech_util::formatJsonFromString(
resultJson["data"].asString());
handleRecognitionResult(dataJson);
}
}
} else {
ai_engine::core_ai::speech::RecognitionResult result{};
result.success = false;
int errorCode = int(
xunfei_speech_server_error::realtimeAsrErrorCode2speechResult(code)
.errorCode);
Json::Value value = xunfei_speech_util::stringToJson(message);
std::string errorMessage;
if (value.isMember("desc") && value["desc"].isString()) {
errorMessage = value["desc"].asString();
}
// 返回10700错误码(引擎错误)时包含'超过15秒没有向服务端发送音频数据'的情况
// 会在报错信息中显示, 所以需要单独判断一下,返回正确的错误码
if (errorMessage.find("37005") != std::string::npos) {
errorCode = (int)ai_engine::SpeechEngineErrorCode::TooLittleData;
}
result.error = EngineError(errorCode, errorMessage);
result.modelName = currentModelName_;
result.text = "";
result.speakerId = -1;
result.reason =
ai_engine::core_ai::speech::SpeechResultReason::SpeechErrorOccurred;
if (recognitionCallback_) {
recognitionCallback_(result);
}
}
}
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::initContinuousRecognitionModule() {
ai_engine::result::AiEngineBaseResult result{false, EngineError{}};
std::atomic_bool isOpen{false};
std::atomic_bool isMessaged{false};
XunfeiWebSocketClient client{};
client.onopen = [&isOpen]() {
LOGI("Init continuous asr: on open.");
isOpen = true;
};
client.onmessage = [&isMessaged, &result](const std::string &message) {
LOGI("Init continuous asr: on message:", message, ".");
// {
// "action":"started",
// "code":"0",
// "data":"",
// "desc":"success",
// "sid":"rta13b4a589@dx2f5f1b49a270000100"
// }
int code = xunfei_speech_server_error::parseErrorCode(message);
if (code == -1) {
// 回复消息体不完整
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Init continuous asr on message error");
} else if (code == 0) {
result.success = true;
} else {
result.success = false;
int errorCode = int(
xunfei_speech_server_error::realtimeAsrErrorCode2speechResult(
code)
.errorCode);
Json::Value value = xunfei_speech_util::stringToJson(message);
std::string errorMessage;
if (value.isMember("desc") && value["desc"].isString()) {
errorMessage = value["desc"].asString();
}
result.error = EngineError(errorCode, errorMessage);
}
isMessaged = true;
};
client.onclose = []() { LOGI("Init continuous asr: on close."); };
client.setPingInterval(10000);
std::string url = continuousAsrBuildUrl();
if (url.empty()) {
// url为空,是由于时间服务器连不上,所以获取不到,报网络断开错误
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Get NTP server error.");
return result;
}
int ret = client.open(url.c_str());
if (ret != 0) {
LOGI("Init continuous asr: open failed.");
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Init continuous asr: open failed.");
return result;
}
client.setSendTimeoutSec(5);
client.setRecvTimeoutSec(5);
for (int i = 0; i < 50; ++i) {
// 100ms
usleep(100 * 1000);
if (isOpen && isMessaged) {
break;
}
}
if (!result.success && result.error.getCode() == -1) {
// 不成功且错误码为默认值时
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Init continuous asr: time out.");
}
return result;
}
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::startContinuousRecognition(
const std::string ¶ms) {
std::lock_guard locker(mutex_);
LOGD("Continuous asr: start.");
ai_engine::result::AiEngineBaseResult result{false, EngineError{}};
setRecognitionParams(params);
wssClient_.onopen = [this]() {
LOGI("Continuous asr: on open.");
continuousAsrOpened_ = true;
};
wssClient_.onmessage = [this](const std::string &message) {
handleMessage(message);
};
wssClient_.onclose = [this]() { LOGI("Continuous asr: on close."); };
wssClient_.setPingInterval(10000);
std::string url = continuousAsrBuildUrl();
LOGD(url);
if (url.empty()) {
// url为空,是由于时间服务器连不上,所以获取不到,报网络断开错误
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Get NTP server error.");
return result;
}
continuousAsrOpened_ = false;
int ret = wssClient_.open(url.c_str());
if (ret != 0) {
LOGI("Continuous asr: open failed.");
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Continuous asr: open failed.");
return result;
}
wssClient_.setSendTimeoutSec(15);
wssClient_.setRecvTimeoutSec(15);
// 需求要求5秒
for (int i = 0; i < 50; ++i) {
// 100ms
usleep(100 * 1000);
if (continuousAsrOpened_) {
break;
}
}
result.success = true;
continuousAsrFinished_ = false;
ai_engine::core_ai::speech::RecognitionResult recognitionResult{};
recognitionResult.success = true;
recognitionResult.error = {};
recognitionResult.modelName = currentModelName_;
recognitionResult.text = "";
recognitionResult.speakerId = 0;
recognitionResult.reason = ai_engine::core_ai::speech::SpeechResultReason::
SpeechRecognitionStarted;
recognitionCallback_(recognitionResult);
return result;
}
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::writeContinuousRecognitionAudioData(
const std::vector &audioData) {
std::lock_guard locker(mutex_);
ai_engine::result::AiEngineBaseResult result{false, EngineError{}};
if (continuousAsrFinished_) {
// 主动关闭
result.success = true;
return result;
}
if (!continuousAsrOpened_) {
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Continuous asr: wss client is not inited.");
return result;
}
if (audioData.empty()) {
result.success = false;
result.error = ai_engine::EngineError(
int(ai_engine::SpeechEngineErrorCode::TooLittleData),
"Continuous asr: audio data is empty.");
return result;
}
int sent =
wssClient_.send(reinterpret_cast(audioData.data()),
int(audioData.size()), ws_opcode::WS_OPCODE_BINARY);
if (sent < 0) {
LOGI("Continuous asr: send failed.");
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Continuous asr: send failed.");
return result;
}
result.success = true;
return result;
}
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::stopContinuousRecognition() {
std::lock_guard locker(mutex_);
wssClient_.send(R"({"end": true})");
continuousAsrFinished_ = true;
return {true, EngineError{}};
}
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::destroyContinuousRecognitionModule() {
wssClient_.close();
return {true, EngineError{}};
}
// 开始 - 一次语音识别私有函数
bool XunfeiSpeechEnginePrivate::onceAsrParamsCheck(
const std::vector &audioData, EngineError &error) {
if (!recognitionCallback_) {
error =
EngineError(int(ai_engine::SpeechEngineErrorCode::TooLittleParams),
"Once Asr: callback is nullptr.");
return false;
}
if (audioData.empty()) {
// 此处不用调用回调函数通知
error =
EngineError(int(ai_engine::SpeechEngineErrorCode::TooLittleData),
"Once Asr: the audio data is empty.");
return false;
}
return true;
}
std::string XunfeiSpeechEnginePrivate::onceAsrBuildData(
const std::string ¶ms, const std::vector &audioData) {
setRecognitionParams(params);
Json::Value bodyJson;
bodyJson["common"]["app_id"] = appIdContinuous_;
bodyJson["business"]["language"] = "zh_cn";
bodyJson["business"]["domain"] = "iat";
bodyJson["business"]["accent"] = "mandarin";
bodyJson["data"]["status"] = 2;
bodyJson["data"]["format"] = "audio/L16;rate=" + std::to_string(rate_);
bodyJson["data"]["encoding"] = "raw";
bodyJson["data"]["audio"] = xunfei_speech_util::base64Encode(
std::string(audioData.begin(), audioData.end()));
return bodyJson.toStyledString();
}
std::string XunfeiSpeechEnginePrivate::onceAsrBuildUrl() {
auto urlData = xunfei_speech_util::parseURL(asrHostUrl_);
std::string date = xunfei_speech_util::getGmtTime();
std::string originSignature;
originSignature += "host: " + urlData["Host"] + "\n";
originSignature += "date: " + date + "\n";
originSignature += "GET " + urlData["Path"] + " HTTP/1.1";
std::string shaSignature = xunfei_speech_util::hmacSha256Encode(
originSignature, secretKeyContinuous_);
std::string authorization;
authorization += R"(hmac username=")" + apiKeyContinuous_ + R"(", )";
authorization += R"(algorithm="hmac-sha256", )";
authorization += R"(headers="host date request-line", )";
authorization += R"(signature=")" + shaSignature + R"(")";
authorization = xunfei_speech_util::base64Encode(authorization);
std::string url;
url += std::string(asrHostUrl_);
url += "?";
url += "authorization=" + xunfei_speech_util::urlEncode(authorization);
url += "&";
url += "date=" + xunfei_speech_util::urlEncode(date);
url += "&";
url += "host=" + urlData["Host"];
return url;
}
bool XunfeiSpeechEnginePrivate::onceAsrOnMessageCheck(
const std::string &message) {
if (int errorCode = xunfei_speech_server_error::parseErrorCode(message)) {
LOGE("Once asr: xunfei recognize failed", message);
int theErrorCode =
int(xunfei_speech_server_error::asrErrorCode2speechResult(errorCode)
.errorCode);
EngineError error(theErrorCode, message);
ai_engine::core_ai::speech::RecognitionResult result{
false,
error,
currentModelName_,
{},
0,
ai_engine::core_ai::speech::SpeechResultReason::
SpeechErrorOccurred};
if (recognitionCallback_) {
recognitionCallback_(result);
}
return false;
}
return true;
}
void XunfeiSpeechEnginePrivate::onceAsrOnMessage(const std::string &message,
std::atomic_bool &isFinish) {
if (!onceAsrOnMessageCheck(message)) {
return;
}
Json::Value recognitionJson =
xunfei_speech_util::formatJsonFromString(message);
EngineError error{};
ai_engine::core_ai::speech::RecognitionResult result;
result.success = true;
result.error = error;
result.modelName = currentModelName_;
result.speakerId = 0;
for (int i = 0; i < recognitionJson["data"]["result"]["ws"].size(); i++) {
for (int j = 0;
j < recognitionJson["data"]["result"]["ws"][i]["cw"].size(); j++) {
result.text +=
recognitionJson["data"]["result"]["ws"][i]["cw"][j]["w"]
.asString();
}
}
result.reason = ai_engine::core_ai::speech::SpeechResultReason::
SpeechRecognitionCompleted;
if (recognitionCallback_) {
recognitionCallback_(result);
}
isFinish = true;
}
// 结束 - 一次语音识别私有函数
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::initRecognizeOnceModule() {
ai_engine::result::AiEngineBaseResult result{false, EngineError{}};
std::atomic_bool isOpen{false};
std::atomic_bool isClose{false};
XunfeiWebSocketClient client{};
client.onopen = [&isOpen]() {
LOGI("Init once asr: on open.");
isOpen = true;
};
client.onmessage = [](const std::string &message) {
LOGI("Init once asr: on message:", message, ".");
};
client.onclose = [&isClose]() {
LOGI("Init once asr: on close.");
isClose = true;
};
client.setPingInterval(10000);
std::string url = onceAsrBuildUrl();
int ret = client.open(url.c_str());
if (ret != 0) {
LOGI("Init once asr: open failed.");
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Init once asr: open failed.");
return result;
}
client.setSendTimeoutSec(15);
client.setRecvTimeoutSec(15);
// 最多等待100次100毫秒(10秒),之后,如果还没有打开,则认为失败。
for (int i = 0; i < 100; ++i) {
// 100ms
usleep(100 * 1000);
if (isOpen || isClose) {
auto response = client.getHttpResponse();
LOGI(response->status_code, response->status_message());
// 401 缺少authorization参数
// {"message": "Unauthorized"}
// 401 签名参数解析失败
// {"message": "HMAC signature cannot be verified"}
// 401 签名校验失败
// {"message": "HMAC signature does not match"}
// 403 时钟偏移校验失败
// {"message": "HMAC signature cannot be verified, a valid
// date or x-date header is required for HMAC Authentication"}
// 403 IP白名单校验失败
// {"message" : "Your IP address is not allowed"}
if (response->status_code == 101) {
// 说明服务端接收到请求,并把协议切换成为websocket协议
result.success = true;
} else if (response->status_code == 400) {
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Init once asr: NetworkDisconnected.");
} else if (response->status_code == 403) {
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::SslCertificateError),
"Init once asr: SslCertificateError.");
} else {
// TODO 先报鉴权错误,后续再进行区分
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::Unauthorized),
"Init once asr: unauthorized.");
}
break;
}
}
if (!result.success && result.error.getCode() == -1) {
// 不成功且错误码为默认值时
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Init once asr: time out.");
}
return result;
}
ai_engine::result::AiEngineBaseResult XunfeiSpeechEnginePrivate::recognizeOnce(
const std::string ¶ms, const std::vector &audioData) {
EngineError engineError{};
if (!onceAsrParamsCheck(audioData, engineError)) {
return {false, engineError};
}
std::string data = onceAsrBuildData(params, audioData);
std::string url = onceAsrBuildUrl();
// 开始 - libhv websocket client准备
std::atomic_bool isOpen{false};
onceAsrFinished_ = false;
XunfeiWebSocketClient client{};
client.onopen = [&isOpen]() {
LOGI("Once asr: on open.");
isOpen = true;
};
client.onmessage = [this](const std::string &message) {
LOGI("Once asr: on message:", message);
onceAsrOnMessage(message, onceAsrFinished_);
};
client.onclose = [&isOpen, this]() {
LOGI("Once asr: on close.");
isOpen = true;
onceAsrFinished_ = true;
};
client.setPingInterval(10000);
// 结束 - libhv websocket client准备
// 开始 - 发送代码
int ret = client.open(url.c_str());
if (ret != 0) {
engineError = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Once asr: open failed.");
return {false, engineError};
}
client.setSendTimeoutSec(15);
client.setRecvTimeoutSec(15);
// 最多等待100次100毫秒(10秒),之后,如果还没有打开,则认为失败。
for (int i = 0; i < 100; ++i) {
// 100ms
usleep(100 * 1000);
if (isOpen) {
break;
}
}
ret = client.send(data.data(), data.size());
if (ret == -1) {
engineError = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Once asr: send failed.");
return {false, engineError};
}
while (!onceAsrFinished_) {
usleep(100 * 1000);
}
client.close();
return {true, EngineError{}};
// 结束 - 发送代码
}
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::destroyRecognizeOnceModule() {
onceAsrFinished_ = true;
return {true, EngineError{}};
}
// 开始 - 一次语音合成私有函数
bool XunfeiSpeechEnginePrivate::onceTtsParamsCheck(const std::string &text,
EngineError &engineError) {
if (!synthesisCallback_) {
engineError =
EngineError(int(ai_engine::SpeechEngineErrorCode::TooLittleParams),
"Once tts: callback is nullptr.");
return false;
}
if (text.empty()) {
// 此处不用调用回调函数通知
engineError =
EngineError(int(ai_engine::SpeechEngineErrorCode::TooLittleData),
"Once tts: text is empty.");
return false;
}
return true;
}
std::string
XunfeiSpeechEnginePrivate::onceTtsBuildData(const std::string ¶ms,
const std::string &text) {
setSynthesisParams(params);
Json::Value bodyJson;
bodyJson["common"]["app_id"] = appIdContinuous_;
bodyJson["business"]["aue"] = "raw";
bodyJson["business"]["sfl"] = 1;
bodyJson["business"]["vcn"] = "xiaoyan";
bodyJson["business"]["speed"] = speed_;
bodyJson["business"]["volume"] = volume_;
bodyJson["business"]["pitch"] = pitch_;
bodyJson["business"]["tte"] = "UTF8";
bodyJson["data"]["status"] = 2;
bodyJson["data"]["text"] = xunfei_speech_util::base64Encode(text);
return bodyJson.toStyledString();
}
std::string XunfeiSpeechEnginePrivate::onceTtsBuildUrl() {
auto urlData = xunfei_speech_util::parseURL(ttsHostUrl_);
std::string date = xunfei_speech_util::getGmtTime();
std::string originSignature;
originSignature += "host: " + urlData["Host"] + "\n";
originSignature += "date: " + date + "\n";
originSignature += "GET " + urlData["Path"] + " HTTP/1.1";
std::string shaSignature = xunfei_speech_util::hmacSha256Encode(
originSignature, secretKeyContinuous_);
std::string authorization;
authorization += R"(hmac username=")" + apiKeyContinuous_ + R"(", )";
authorization += R"(algorithm="hmac-sha256", )";
authorization += R"(headers="host date request-line", )";
authorization += R"(signature=")" + shaSignature + R"(")";
authorization = xunfei_speech_util::base64Encode(authorization);
std::string url;
url += std::string(ttsHostUrl_);
url += "?";
url += "authorization=" + xunfei_speech_util::urlEncode(authorization);
url += "&";
url += "date=" + xunfei_speech_util::urlEncode(date);
url += "&";
url += "host=" + urlData["Host"];
return url;
}
bool XunfeiSpeechEnginePrivate::onceTtsOnMessageCheck(
const std::string &message) {
if (int errorCode = xunfei_speech_server_error::parseErrorCode(message)) {
LOGE("Xunfei synthesize failed", message, ".");
int theErrorCode =
int(xunfei_speech_server_error::ttsErrorCode2speechResult(errorCode)
.errorCode);
EngineError error(theErrorCode, message);
ai_engine::core_ai::speech::SynthesisResult result{
false,
error,
currentModelName_,
{},
{},
{},
{},
ai_engine::core_ai::speech::SpeechResultReason::
SpeechErrorOccurred};
if (synthesisCallback_) {
synthesisCallback_(result);
}
return false;
}
return true;
}
void XunfeiSpeechEnginePrivate::onceTtsOnMessage(const std::string &message,
std::atomic_bool &isFinish) {
if (!onceTtsOnMessageCheck(message)) {
return;
}
Json::Value synthesisJson =
xunfei_speech_util::formatJsonFromString(message);
ai_engine::core_ai::speech::SynthesisResult result;
result.success = true;
result.error = EngineError{};
result.modelName = currentModelName_;
result.audioFormat = "pcm";
result.audioRate = 16000;
result.audioChannel = 1;
result.base64AudioData = synthesisJson["data"]["audio"].asCString();
result.reason =
ai_engine::core_ai::speech::SpeechResultReason::SpeechSynthesizing;
if (synthesisJson["data"]["status"].asInt() == 2) {
result.reason = ai_engine::core_ai::speech::SpeechResultReason::
SpeechSynthesisCompleted;
isFinish = true;
}
if (synthesisCallback_) {
synthesisCallback_(result);
}
}
// 结束 - 一次语音合成私有函数
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::initSynthesizeModule() {
ai_engine::result::AiEngineBaseResult result{false, EngineError{}};
std::atomic_bool isOpen{false};
std::atomic_bool isClose{false};
XunfeiWebSocketClient client{};
client.onopen = [&isOpen]() {
LOGI("Init once tts: on open.");
isOpen = true;
};
client.onmessage = [](const std::string &message) {
LOGI("Init once tts: on message:", message, ".");
};
client.onclose = [&isClose]() {
LOGI("Init once tts: on close.");
isClose = true;
};
client.setPingInterval(10000);
std::string url = onceTtsBuildUrl();
int ret = client.open(url.c_str());
if (ret != 0) {
LOGI("Init once tts: open failed.");
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Init once tts: open failed.");
return result;
}
client.setSendTimeoutSec(15);
client.setRecvTimeoutSec(15);
// 最多等待100次100毫秒(10秒),之后,如果还没有打开,则认为失败。
for (int i = 0; i < 100; ++i) {
// 100ms
usleep(100 * 1000);
if (isOpen || isClose) {
auto response = client.getHttpResponse();
LOGI(response->status_code, response->status_message());
// 401 缺少authorization参数
// {"message": "Unauthorized"}
// 401 签名参数解析失败
// {"message": "HMAC signature cannot be verified"}
// 401 签名校验失败
// {"message": "HMAC signature does not match"}
// 403 时钟偏移校验失败
// {"message": "HMAC signature cannot be verified, a valid
// date or x-date header is required for HMAC Authentication"}
// 403 IP白名单校验失败
// {"message" : "Your IP address is not allowed"}
if (response->status_code == 101) {
// 说明服务端接收到请求,并把协议切换成为websocket协议
result.success = true;
} else if (response->status_code == 400) {
// TODO 临时映射到SslCertificateError错误,后续完善
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::SslCertificateError),
"Init once tts: SslCertificateError.");
} else if (response->status_code == 403) {
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::SslCertificateError),
"Init once tts: SslCertificateError.");
} else {
// TODO 先报鉴权错误,后续再进行区分
result.success = false;
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::Unauthorized),
"Init once tts: unauthorized");
}
break;
}
}
if (!result.success && result.error.getCode() == -1) {
// 不成功且错误码为默认值时
result.error = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Init once tts: time out.");
}
return result;
}
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::synthesizeText(const std::string ¶ms,
const std::string &text) {
EngineError engineError{};
if (!onceTtsParamsCheck(text, engineError)) {
return {false, engineError};
}
std::string data = onceTtsBuildData(params, text);
std::string url = onceTtsBuildUrl();
// 开始 - libhv websocket client准备
std::atomic_bool isOpen{false};
std::atomic_bool isClose{false};
onceTtsFinished_ = false;
XunfeiWebSocketClient client{};
client.onopen = [&isOpen]() {
LOGI("Once tts: on open.");
isOpen = true;
};
client.onmessage = [this](const std::string &message) {
// LOGI("Once tts: on message:", message, ".");
onceTtsOnMessage(message, onceTtsFinished_);
};
client.onclose = [&isClose, this]() {
LOGI("Once tts: on close.");
isClose = true;
onceTtsFinished_ = true;
};
client.setPingInterval(10000);
// 结束 - libhv websocket client准备
ai_engine::core_ai::speech::SynthesisResult result{
true,
EngineError{},
currentModelName_,
{},
{},
{},
{},
ai_engine::core_ai::speech::SpeechResultReason::SpeechSynthesisStarted};
if (synthesisCallback_) {
synthesisCallback_(result);
}
// 开始 - 发送代码
int ret = client.open(url.c_str());
LOGD("Once tts: open status:", ret);
if (ret != 0) {
engineError = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Once tts: open failed.");
return {false, engineError};
}
client.setSendTimeoutSec(15);
client.setRecvTimeoutSec(15);
// 等待打开后再发送数据
for (int i = 0; i < 100; ++i) {
// 100ms
usleep(100 * 1000);
if (isOpen || isClose) {
break;
}
}
// 如果关闭了,说明打开失败
if (isClose) {
HttpResponsePtr response = client.getHttpResponse();
LOGD(response->status_code, response->body);
if (response != nullptr && response->status_code == 400) {
engineError = EngineError(
int(ai_engine::SpeechEngineErrorCode::SslCertificateError),
"Once tts: open failed.");
return {false, engineError};
}
}
ret = client.send(data);
if (ret == -1) {
engineError = EngineError(
int(ai_engine::SpeechEngineErrorCode::NetworkDisconnected),
"Once tts: send failed.");
return {false, engineError};
}
while (!onceTtsFinished_) {
usleep(100 * 1000);
}
client.close();
return {true, EngineError{}};
// 结束 - 发送代码
}
ai_engine::result::AiEngineBaseResult
XunfeiSpeechEnginePrivate::destroySynthesizeModule() {
onceTtsFinished_ = true;
return {true, EngineError{}};
}