|
|
@@ -0,0 +1,421 @@
|
|
|
+package com.telerobot.fs.tts.xfyun;
|
|
|
+
|
|
|
+import com.alibaba.fastjson.JSON;
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.telerobot.fs.config.CallConfig;
|
|
|
+import com.telerobot.fs.config.SystemConfig;
|
|
|
+import com.telerobot.fs.utils.WaveHeader;
|
|
|
+import okhttp3.OkHttpClient;
|
|
|
+import okhttp3.Request;
|
|
|
+import okhttp3.Response;
|
|
|
+import okhttp3.WebSocket;
|
|
|
+import okhttp3.WebSocketListener;
|
|
|
+import okio.ByteString;
|
|
|
+import org.apache.commons.lang.StringUtils;
|
|
|
+import org.dom4j.Document;
|
|
|
+import org.dom4j.Element;
|
|
|
+import org.dom4j.io.SAXReader;
|
|
|
+import org.slf4j.Logger;
|
|
|
+import org.slf4j.LoggerFactory;
|
|
|
+
|
|
|
+import javax.crypto.Mac;
|
|
|
+import javax.crypto.spec.SecretKeySpec;
|
|
|
+import javax.net.ssl.SSLContext;
|
|
|
+import javax.net.ssl.TrustManager;
|
|
|
+import javax.net.ssl.X509TrustManager;
|
|
|
+import java.io.ByteArrayOutputStream;
|
|
|
+import java.io.File;
|
|
|
+import java.io.FileOutputStream;
|
|
|
+import java.io.IOException;
|
|
|
+import java.net.URLEncoder;
|
|
|
+import java.nio.charset.StandardCharsets;
|
|
|
+import java.security.KeyManagementException;
|
|
|
+import java.security.NoSuchAlgorithmException;
|
|
|
+import java.security.SecureRandom;
|
|
|
+import java.time.ZoneId;
|
|
|
+import java.time.ZonedDateTime;
|
|
|
+import java.time.format.DateTimeFormatter;
|
|
|
+import java.util.Base64;
|
|
|
+import java.util.List;
|
|
|
+import java.util.UUID;
|
|
|
+import java.util.concurrent.CountDownLatch;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.concurrent.atomic.AtomicReference;
|
|
|
+
|
|
|
+public class XfyunCloneTtsFileSynthesizer {
|
|
|
+ private static final Logger logger = LoggerFactory.getLogger(XfyunCloneTtsFileSynthesizer.class);
|
|
|
+ private static final String CLONE_TTS_WS_URL = "wss://cn-huabei-1.xf-yun.com/v1/private/voice_clone";
|
|
|
+ private static final String CLONE_TTS_HOST = "cn-huabei-1.xf-yun.com";
|
|
|
+ private static final String CLONE_TTS_PATH = "/v1/private/voice_clone";
|
|
|
+ private static final int DEFAULT_SAMPLE_RATE = 8000;
|
|
|
+ private static final int DEFAULT_TIMEOUT_SECONDS = 45;
|
|
|
+
|
|
|
+ private XfyunCloneTtsFileSynthesizer() {
|
|
|
+ }
|
|
|
+
|
|
|
+ public static String synthesizeToWavFile(String traceId, String assetId, String text) {
|
|
|
+ if (StringUtils.isBlank(assetId) || StringUtils.isBlank(text)) {
|
|
|
+ return "";
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ CloneConfig config = loadCloneConfig();
|
|
|
+ byte[] audioBytes = requestCloneAudio(config, assetId.trim(), text);
|
|
|
+ if (audioBytes.length == 0) {
|
|
|
+ logger.warn("{} xfyun clone tts returned empty audio.", traceId);
|
|
|
+ return "";
|
|
|
+ }
|
|
|
+ String wavPath = buildWavFilePath(traceId);
|
|
|
+ writePcmAsWav(wavPath, audioBytes, config.sampleRate);
|
|
|
+ logger.info("{} xfyun clone tts synthesized wav file={}, bytes={}, sampleRate={}",
|
|
|
+ traceId, wavPath, audioBytes.length, config.sampleRate);
|
|
|
+ return wavPath;
|
|
|
+ } catch (Throwable e) {
|
|
|
+ logger.error("{} xfyun clone tts synthesize failed, assetId={}, err={}",
|
|
|
+ traceId,
|
|
|
+ assetId,
|
|
|
+ e.toString(),
|
|
|
+ e);
|
|
|
+ return "";
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static byte[] requestCloneAudio(CloneConfig config, String assetId, String text) throws Exception {
|
|
|
+ OkHttpClient client = config.verifyPeer ? createDefaultClient() : createUnsafeOkHttpClient();
|
|
|
+ String authUrl = buildCloneWebsocketUrl(config.apiKey, config.apiSecret);
|
|
|
+ CountDownLatch latch = new CountDownLatch(1);
|
|
|
+ AtomicReference<String> errRef = new AtomicReference<String>("");
|
|
|
+ ByteArrayOutputStream audioOutput = new ByteArrayOutputStream();
|
|
|
+
|
|
|
+ Request request = new Request.Builder().url(authUrl).build();
|
|
|
+ client.newWebSocket(request, new WebSocketListener() {
|
|
|
+ @Override
|
|
|
+ public void onOpen(WebSocket webSocket, Response response) {
|
|
|
+ JSONObject payload = buildCloneTtsPayload(config, assetId, text);
|
|
|
+ webSocket.send(payload.toJSONString());
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void onMessage(WebSocket webSocket, String textMessage) {
|
|
|
+ try {
|
|
|
+ JSONObject json = JSON.parseObject(textMessage);
|
|
|
+ int code = parseResponseCode(json);
|
|
|
+ if (code != 0) {
|
|
|
+ errRef.set(buildResponseError(json));
|
|
|
+ webSocket.close(1000, "error");
|
|
|
+ latch.countDown();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ JSONObject payload = json.getJSONObject("payload");
|
|
|
+ if (payload == null) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ JSONObject audio = payload.getJSONObject("audio");
|
|
|
+ if (audio == null) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ String audioChunk = audio.getString("audio");
|
|
|
+ if (StringUtils.isNotBlank(audioChunk)) {
|
|
|
+ audioOutput.write(Base64.getDecoder().decode(audioChunk));
|
|
|
+ }
|
|
|
+
|
|
|
+ if (audio.getIntValue("status") == 2) {
|
|
|
+ webSocket.close(1000, "done");
|
|
|
+ latch.countDown();
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ errRef.set("parse clone tts response failed: " + e.getMessage());
|
|
|
+ webSocket.close(1000, "parse-error");
|
|
|
+ latch.countDown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void onMessage(WebSocket webSocket, ByteString bytes) {
|
|
|
+ onMessage(webSocket, bytes.utf8());
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void onFailure(WebSocket webSocket, Throwable t, Response response) {
|
|
|
+ errRef.set(buildWebsocketFailureMessage(t, response));
|
|
|
+ latch.countDown();
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void onClosed(WebSocket webSocket, int code, String reason) {
|
|
|
+ latch.countDown();
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ if (!latch.await(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
|
|
|
+ throw new IOException("xfyun clone tts timeout");
|
|
|
+ }
|
|
|
+ if (StringUtils.isNotBlank(errRef.get())) {
|
|
|
+ throw new IOException(errRef.get());
|
|
|
+ }
|
|
|
+ return audioOutput.toByteArray();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static JSONObject buildCloneTtsPayload(CloneConfig config, String assetId, String text) {
|
|
|
+ JSONObject root = new JSONObject(true);
|
|
|
+
|
|
|
+ JSONObject header = new JSONObject(true);
|
|
|
+ header.put("app_id", config.appId);
|
|
|
+ header.put("status", 2);
|
|
|
+ header.put("res_id", assetId);
|
|
|
+ root.put("header", header);
|
|
|
+
|
|
|
+ JSONObject audio = new JSONObject(true);
|
|
|
+ audio.put("encoding", "raw");
|
|
|
+ audio.put("sample_rate", config.sampleRate);
|
|
|
+
|
|
|
+ JSONObject tts = new JSONObject(true);
|
|
|
+ tts.put("vcn", config.cloneVcn);
|
|
|
+ tts.put("volume", 50);
|
|
|
+ tts.put("rhy", 0);
|
|
|
+ tts.put("pybuffer", 1);
|
|
|
+ tts.put("speed", 50);
|
|
|
+ tts.put("pitch", 50);
|
|
|
+ tts.put("bgs", 0);
|
|
|
+ tts.put("reg", 0);
|
|
|
+ tts.put("rdn", 0);
|
|
|
+ tts.put("audio", audio);
|
|
|
+
|
|
|
+ JSONObject parameter = new JSONObject(true);
|
|
|
+ parameter.put("tts", tts);
|
|
|
+ root.put("parameter", parameter);
|
|
|
+
|
|
|
+ JSONObject textNode = new JSONObject(true);
|
|
|
+ textNode.put("encoding", "utf8");
|
|
|
+ textNode.put("compress", "raw");
|
|
|
+ textNode.put("format", "plain");
|
|
|
+ textNode.put("status", 2);
|
|
|
+ textNode.put("seq", 0);
|
|
|
+ textNode.put("text", Base64.getEncoder().encodeToString(text.getBytes(StandardCharsets.UTF_8)));
|
|
|
+
|
|
|
+ JSONObject payload = new JSONObject(true);
|
|
|
+ payload.put("text", textNode);
|
|
|
+ root.put("payload", payload);
|
|
|
+ return root;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static CloneConfig loadCloneConfig() throws Exception {
|
|
|
+ String fsConfDirectory = SystemConfig.getValue("fs_conf_directory");
|
|
|
+ if (StringUtils.isBlank(fsConfDirectory)) {
|
|
|
+ throw new IllegalStateException("fs_conf_directory is empty");
|
|
|
+ }
|
|
|
+
|
|
|
+ File confFile = new File(fsConfDirectory, "autoload_configs/xf_tts.conf.xml");
|
|
|
+ if (!confFile.exists()) {
|
|
|
+ throw new IllegalStateException("xf_tts.conf.xml not found: " + confFile.getAbsolutePath());
|
|
|
+ }
|
|
|
+
|
|
|
+ SAXReader reader = new SAXReader();
|
|
|
+ Document document = reader.read(confFile);
|
|
|
+ List<Element> params = new java.util.ArrayList<Element>();
|
|
|
+ collectElementsByName(document.getRootElement(), "param", params);
|
|
|
+
|
|
|
+ CloneConfig config = new CloneConfig();
|
|
|
+ for (Element element : params) {
|
|
|
+ String name = StringUtils.trimToEmpty(element.attributeValue("name"));
|
|
|
+ String value = StringUtils.trimToEmpty(element.attributeValue("value"));
|
|
|
+ if ("app-id".equalsIgnoreCase(name)) {
|
|
|
+ config.appId = value;
|
|
|
+ } else if ("api-key".equalsIgnoreCase(name)) {
|
|
|
+ config.apiKey = value;
|
|
|
+ } else if ("api-secret".equalsIgnoreCase(name)) {
|
|
|
+ config.apiSecret = value;
|
|
|
+ } else if ("sample-rate".equalsIgnoreCase(name)) {
|
|
|
+ config.sampleRate = parseSampleRate(value);
|
|
|
+ } else if ("verify-peer".equalsIgnoreCase(name)) {
|
|
|
+ config.verifyPeer = !"false".equalsIgnoreCase(value);
|
|
|
+ } else if ("clone-vcn".equalsIgnoreCase(name)) {
|
|
|
+ config.cloneVcn = value;
|
|
|
+ } else if ("clone-engine-version".equalsIgnoreCase(name) && "omni_v1".equalsIgnoreCase(value)) {
|
|
|
+ config.cloneVcn = "x6_clone";
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (StringUtils.isBlank(config.appId)
|
|
|
+ || StringUtils.isBlank(config.apiKey)
|
|
|
+ || StringUtils.isBlank(config.apiSecret)) {
|
|
|
+ throw new IllegalStateException("xf_tts.conf.xml is missing clone account credentials");
|
|
|
+ }
|
|
|
+ if (StringUtils.isBlank(config.cloneVcn)) {
|
|
|
+ config.cloneVcn = "x5_clone";
|
|
|
+ }
|
|
|
+ return config;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void collectElementsByName(Element element, String name, List<Element> result) {
|
|
|
+ if (element == null) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ if (name.equalsIgnoreCase(element.getName())) {
|
|
|
+ result.add(element);
|
|
|
+ }
|
|
|
+ List<Element> children = element.elements();
|
|
|
+ for (Element child : children) {
|
|
|
+ collectElementsByName(child, name, result);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static int parseSampleRate(String value) {
|
|
|
+ try {
|
|
|
+ int sampleRate = Integer.parseInt(StringUtils.trimToEmpty(value));
|
|
|
+ return sampleRate > 0 ? sampleRate : DEFAULT_SAMPLE_RATE;
|
|
|
+ } catch (Exception e) {
|
|
|
+ return DEFAULT_SAMPLE_RATE;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static String buildCloneWebsocketUrl(String apiKey, String apiSecret) throws Exception {
|
|
|
+ String date = DateTimeFormatter.RFC_1123_DATE_TIME.format(ZonedDateTime.now(ZoneId.of("GMT")));
|
|
|
+ String signatureOrigin = "host: " + CLONE_TTS_HOST + "\n"
|
|
|
+ + "date: " + date + "\n"
|
|
|
+ + "GET " + CLONE_TTS_PATH + " HTTP/1.1";
|
|
|
+ Mac mac = Mac.getInstance("HmacSHA256");
|
|
|
+ mac.init(new SecretKeySpec(apiSecret.getBytes(StandardCharsets.UTF_8), "HmacSHA256"));
|
|
|
+ String signature = Base64.getEncoder().encodeToString(mac.doFinal(signatureOrigin.getBytes(StandardCharsets.UTF_8)));
|
|
|
+ String authorizationOrigin = String.format(
|
|
|
+ "api_key=\"%s\", algorithm=\"hmac-sha256\", headers=\"host date request-line\", signature=\"%s\"",
|
|
|
+ apiKey,
|
|
|
+ signature
|
|
|
+ );
|
|
|
+ String authorization = Base64.getEncoder().encodeToString(authorizationOrigin.getBytes(StandardCharsets.UTF_8));
|
|
|
+ return CLONE_TTS_WS_URL
|
|
|
+ + "?authorization=" + urlEncode(authorization)
|
|
|
+ + "&date=" + urlEncode(date)
|
|
|
+ + "&host=" + urlEncode(CLONE_TTS_HOST);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static String buildWavFilePath(String traceId) {
|
|
|
+ String recordRoot = StringUtils.defaultIfEmpty(CallConfig.RECORDINGS_PATH, ".");
|
|
|
+ File targetDir = new File(recordRoot, "xf_tts_clone");
|
|
|
+ if (!targetDir.exists()) {
|
|
|
+ synchronized (targetDir.getAbsolutePath().intern()) {
|
|
|
+ if (!targetDir.exists()) {
|
|
|
+ targetDir.mkdirs();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ String safeTraceId = StringUtils.defaultIfEmpty(traceId, "xf_clone").replaceAll("[^0-9A-Za-z_-]", "_");
|
|
|
+ String fileName = safeTraceId + "_" + System.currentTimeMillis() + "_" + UUID.randomUUID().toString().replace("-", "") + ".wav";
|
|
|
+ return new File(targetDir, fileName).getAbsolutePath();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void writePcmAsWav(String filePath, byte[] pcmBytes, int sampleRate) throws IOException {
|
|
|
+ File targetFile = new File(filePath);
|
|
|
+ File parentDir = targetFile.getParentFile();
|
|
|
+ if (parentDir != null && !parentDir.exists()) {
|
|
|
+ synchronized (parentDir.getAbsolutePath().intern()) {
|
|
|
+ if (!parentDir.exists()) {
|
|
|
+ parentDir.mkdirs();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ WaveHeader header = new WaveHeader();
|
|
|
+ header.fileLength = pcmBytes.length + 36;
|
|
|
+ header.FmtHdrLeth = 16;
|
|
|
+ header.FormatTag = 1;
|
|
|
+ header.Channels = 1;
|
|
|
+ header.SamplesPerSec = sampleRate;
|
|
|
+ header.BitsPerSample = 16;
|
|
|
+ header.BlockAlign = (short) (header.Channels * header.BitsPerSample / 8);
|
|
|
+ header.AvgBytesPerSec = header.BlockAlign * header.SamplesPerSec;
|
|
|
+ header.DataHdrLeth = pcmBytes.length;
|
|
|
+
|
|
|
+ try (FileOutputStream outputStream = new FileOutputStream(targetFile)) {
|
|
|
+ outputStream.write(header.getHeader());
|
|
|
+ outputStream.write(pcmBytes);
|
|
|
+ outputStream.flush();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static int parseResponseCode(JSONObject json) {
|
|
|
+ if (json == null) {
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+ if (json.containsKey("code")) {
|
|
|
+ return json.getIntValue("code");
|
|
|
+ }
|
|
|
+ JSONObject header = json.getJSONObject("header");
|
|
|
+ return header == null ? -1 : header.getIntValue("code");
|
|
|
+ }
|
|
|
+
|
|
|
+ private static String buildResponseError(JSONObject json) {
|
|
|
+ JSONObject header = json == null ? null : json.getJSONObject("header");
|
|
|
+ if (header != null) {
|
|
|
+ return "xfyun clone tts error: code=" + header.getIntValue("code")
|
|
|
+ + ", message=" + header.getString("message")
|
|
|
+ + ", body=" + json.toJSONString();
|
|
|
+ }
|
|
|
+ return "xfyun clone tts error: " + (json == null ? "" : json.toJSONString());
|
|
|
+ }
|
|
|
+
|
|
|
+ private static String buildWebsocketFailureMessage(Throwable throwable, Response response) {
|
|
|
+ StringBuilder sb = new StringBuilder("xfyun clone websocket failed");
|
|
|
+ if (response != null) {
|
|
|
+ sb.append(": http ").append(response.code());
|
|
|
+ }
|
|
|
+ if (throwable != null && StringUtils.isNotBlank(throwable.getMessage())) {
|
|
|
+ sb.append(", ").append(throwable.getMessage());
|
|
|
+ }
|
|
|
+ return sb.toString();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static String urlEncode(String value) throws Exception {
|
|
|
+ return URLEncoder.encode(value, "UTF-8");
|
|
|
+ }
|
|
|
+
|
|
|
+ private static OkHttpClient createDefaultClient() {
|
|
|
+ return new OkHttpClient.Builder()
|
|
|
+ .connectTimeout(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
|
|
|
+ .readTimeout(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
|
|
|
+ .writeTimeout(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
|
|
|
+ .build();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static OkHttpClient createUnsafeOkHttpClient() {
|
|
|
+ try {
|
|
|
+ final TrustManager[] trustAllCerts = new TrustManager[]{
|
|
|
+ new X509TrustManager() {
|
|
|
+ @Override
|
|
|
+ public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType) {
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType) {
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
|
|
|
+ return new java.security.cert.X509Certificate[0];
|
|
|
+ }
|
|
|
+ }
|
|
|
+ };
|
|
|
+ SSLContext sslContext = SSLContext.getInstance("SSL");
|
|
|
+ sslContext.init(null, trustAllCerts, new SecureRandom());
|
|
|
+ return new OkHttpClient.Builder()
|
|
|
+ .connectTimeout(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
|
|
|
+ .readTimeout(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
|
|
|
+ .writeTimeout(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
|
|
|
+ .sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustAllCerts[0])
|
|
|
+ .hostnameVerifier((hostname, session) -> true)
|
|
|
+ .build();
|
|
|
+ } catch (NoSuchAlgorithmException | KeyManagementException e) {
|
|
|
+ throw new IllegalStateException("create unsafe okhttp client failed", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static class CloneConfig {
|
|
|
+ private String appId;
|
|
|
+ private String apiKey;
|
|
|
+ private String apiSecret;
|
|
|
+ private int sampleRate = DEFAULT_SAMPLE_RATE;
|
|
|
+ private boolean verifyPeer = true;
|
|
|
+ private String cloneVcn = "x5_clone";
|
|
|
+ }
|
|
|
+}
|
