opensource/Chat2DB
1
0
Fork 0
mirror of https://github.com/CodePhiliaX/Chat2DB.git synced 2025-07-30 03:03:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove es

Browse Source
This commit is contained in:
robin
2023-10-28 10:20:09 +08:00
parent 6bd097798e
commit 3f357b93ff
11 changed files with 8 additions and 396390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
chat2db-server/chat2db-server-web/chat2db-server-web-api/pom.xml
View File

@ -84,11 +84,6 @@
</dependency>
<dependency>
<groupId>org.ansj</groupId>
<artifactId>ansj_seg</artifactId>
</dependency>
</dependencies>
</project>

13
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/ChatController.java
View File

@ -45,7 +45,6 @@ import ai.chat2db.server.web.api.http.request.WhiteListRequest;
import ai.chat2db.server.web.api.http.response.EsTableSchemaResponse;
import ai.chat2db.server.web.api.http.response.TableSchemaResponse;
import ai.chat2db.server.web.api.util.ApplicationContextUtil;
import ai.chat2db.server.web.api.util.SegmentUtils;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson2.JSON;
@ -556,10 +555,9 @@ public class ChatController {
if (StringUtils.isNotBlank(apiKey)) {
boolean res = gatewayClientService.checkInWhite(new WhiteListRequest(apiKey, WhiteListTypeEnum.VECTOR.getCode())).getData();
if (res) {
properties = queryDatabaseSchema(queryRequest) + querySchemaByEs(queryRequest);
// properties = queryDatabaseSchema(queryRequest) + querySchemaByEs(queryRequest);
properties = queryDatabaseSchema(queryRequest);
}
} else {
properties = querySchemaByEs(queryRequest);
}
return properties;
}
@ -573,10 +571,11 @@ public class ChatController {
*/
public String queryDatabaseSchema(ChatQueryRequest queryRequest) {
// request embedding
String input = SegmentUtils.baseAnalysis(queryRequest.getMessage());
log.info("search message:{}", input);
FastChatEmbeddingResponse response = distributeAIEmbedding(input);
FastChatEmbeddingResponse response = distributeAIEmbedding(queryRequest.getMessage());
List<List<BigDecimal>> contentVector = new ArrayList<>();
if (Objects.isNull(response) || CollectionUtils.isEmpty(response.getData())) {
return "";
}
contentVector.add(response.getData().get(0).getEmbedding());
// search embedding

2
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDdlController.java
View File

@ -78,7 +78,7 @@ public class RdbDdlController extends EmbeddingController {
try {
Chat2DBContext.putContext(connectInfo);
syncTableVector(request);
syncTableEs(request);
// syncTableEs(request);
} catch (Exception e) {
log.error("sync table vector error", e);
} finally {

2
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java
View File

@ -69,7 +69,7 @@ public class TableController extends EmbeddingController {
try {
Chat2DBContext.putContext(connectInfo);
syncTableVector(request);
syncTableEs(request);
// syncTableEs(request);
} catch (Exception e) {
log.error("sync table vector error", e);
} finally {

94
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/util/SegmentUtils.java
View File

@ -1,94 +0,0 @@
package ai.chat2db.server.web.api.util;
import lombok.extern.slf4j.Slf4j;
import org.ansj.domain.Result;
import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.BaseAnalysis;
import org.ansj.splitWord.analysis.NlpAnalysis;
import org.ansj.splitWord.analysis.ToAnalysis;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
public class SegmentUtils {
/**
* BaseAnalysis
*
* @param content
*/
public static String baseAnalysis(String content) {
Result result = BaseAnalysis.parse(delHTMLTag(content).replace("\n", "").replace(" ", "").replace("\t", ""));
log.info("base analysis result:" + result);
return convertResToString(result);
}
/**
* ToAnalysis
*
* @param content
*/
public static String toAnalysis(String content) {
Result result = ToAnalysis.parse(content);
log.info("to analysis result:" + result);
return convertResToString(result);
}
/**
* NlpAnalysis
*
* @param content
*/
public static String nlpAnalysis(String content) {
Result result = NlpAnalysis.parse(delHTMLTag(content).replace("\n", "").replace(" ", "").replace("\t", ""));
log.info("nlp analysis result:" + result);
return convertResToString(result);
}
/**
* convert result to string
*
* @param result
* @return
*/
private static String convertResToString(Result result) {
List<Term> terms = result.getTerms();
StringBuilder sb = new StringBuilder();
for (Term term : terms) {
String name = term.getName();
String nature = term.getNatureStr();
if (nature.equals("nt") || nature.equals("nr") || nature.equals("n")) {
sb.append(name).append(" ");
}
}
return sb.toString();
}
/**
* delete html tag
*
* @param htmlStr
* @return
*/
public static String delHTMLTag(String htmlStr) {
String regEx_script = "<script[^>]*?>[\\s\\S]*?<\\/script>";
String regEx_style = "<style[^>]*?>[\\s\\S]*?<\\/style>";
String regEx_html = "<[^>]+>";
Pattern p_script = Pattern.compile(regEx_script, Pattern.CASE_INSENSITIVE);
Matcher m_script = p_script.matcher(htmlStr);
htmlStr = m_script.replaceAll("");
Pattern p_style = Pattern.compile(regEx_style, Pattern.CASE_INSENSITIVE);
Matcher m_style = p_style.matcher(htmlStr);
htmlStr = m_style.replaceAll("");
Pattern p_html = Pattern.compile(regEx_html, Pattern.CASE_INSENSITIVE);
Matcher m_html = p_html.matcher(htmlStr);
htmlStr = m_html.replaceAll("");
return htmlStr.trim();
}
}