opensource/Chat2DB
1
0
Fork 0
mirror of https://github.com/CodePhiliaX/Chat2DB.git synced 2025-09-21 12:03:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add CacheManage

Browse Source
This commit is contained in:
jipengfei-jpf
2023-07-08 19:02:53 +08:00
parent 4f5a58e77b
commit 8fd8d43758
11 changed files with 215 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/TablePageQueryParam.java
View File

@ -1,11 +1,7 @@
package ai.chat2db.server.domain.api.param;
import java.io.Serial;
import jakarta.validation.constraints.NotNull;
import ai.chat2db.server.tools.base.wrapper.param.PageQueryParam;
import jakarta.validation.constraints.NotNull;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -44,4 +40,11 @@ public class TablePageQueryParam extends PageQueryParam {
*
*/
private String schemaName;
/**
* if true, refresh the cache
*/
private boolean refresh;
}

21
chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/cache/CacheKey.java vendored Normal file
View File

@ -0,0 +1,21 @@
package ai.chat2db.server.domain.core.cache;
import org.springframework.util.StringUtils;
public class CacheKey {
public static String getDataSourceKey(Long dataSourceId) {
return "schemas_datasourceId_" + dataSourceId;
}
public static String getTableKey(Long dataSourceId, String databaseName, String schemaName) {
StringBuffer stringBuffer = new StringBuffer("tables_dataSourceId" + dataSourceId);
if (!StringUtils.isEmpty(databaseName)) {
stringBuffer.append("_databaseName" + databaseName);
}
if (!StringUtils.isEmpty(schemaName)) {
stringBuffer.append("_schemaName" + schemaName);
}
return stringBuffer.toString();
}
}

51
chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/cache/CacheManage.java vendored
View File

@ -1,6 +1,8 @@
package ai.chat2db.server.domain.core.cache;
import java.io.File;
import java.util.List;
import java.util.function.Function;
import com.alibaba.fastjson2.JSON;
@ -26,21 +28,62 @@ public class CacheManage {
.with(CacheManagerBuilder.persistence(PATH)) // 确保这个路径是存在且有写权限的
.withCache(CACHE, CacheConfigurationBuilder.newCacheConfigurationBuilder(String.class, String.class,
ResourcePoolsBuilder.newResourcePoolsBuilder()
.heap(1, EntryUnit.ENTRIES)
.heap(1000, EntryUnit.ENTRIES)
.disk(20, MemoryUnit.GB, true))) // 磁盘持久化设置为true
.build(true);
}
public static <T> T get(String key,Class<T> clazz) {
public static <T> T get(String key, Class<T> clazz) {
Cache<String, String> myCache = cacheManager.getCache(CACHE, String.class, String.class);
String value = myCache.get(key);
if(!StringUtils.isEmpty(value)){
return JSON.parseObject(value,clazz);
if (!StringUtils.isEmpty(value)) {
return JSON.parseObject(value, clazz);
}
return null;
}
public static <T> List<T> getList(String key, Class<T> clazz) {
Cache<String, String> myCache = cacheManager.getCache(CACHE, String.class, String.class);
String value = myCache.get(key);
if (!StringUtils.isEmpty(value)) {
return JSON.parseArray(value, clazz);
}
return null;
}
public static <T> T get(String key, Class<T> clazz, Function<Object, Boolean> refresh,
Function<Object, T> function) {
T t;
if (refresh.apply(key)) {
t = function.apply(key);
put(key, t);
} else {
t = get(key, clazz);
if (t == null) {
t = function.apply(key);
put(key, t);
}
}
return t;
}
public static <T> List<T> getList(String key, Class<T> clazz, Function<Object, Boolean> refresh,
Function<Object, List<T>> function) {
List<T> t;
if (refresh.apply(key)) {
t = function.apply(key);
put(key, t);
} else {
t = getList(key, clazz);
if (t == null) {
t = function.apply(key);
put(key, t);
}
}
return t;
}
public static void put(String s, Object value) {
Cache<String, String> myCache = cacheManager.getCache(CACHE, String.class, String.class);

41
chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java
View File

@ -4,8 +4,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.alibaba.fastjson2.JSON;
import ai.chat2db.server.domain.api.param.DatabaseOperationParam;
import ai.chat2db.server.domain.api.param.DatabaseQueryAllParam;
import ai.chat2db.server.domain.api.param.MetaDataQueryParam;
@ -20,10 +18,11 @@ import ai.chat2db.spi.model.Database;
import ai.chat2db.spi.model.MetaSchema;
import ai.chat2db.spi.model.Schema;
import ai.chat2db.spi.sql.Chat2DBContext;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import static ai.chat2db.server.domain.core.cache.CacheKey.getDataSourceKey;
/**
* @author moji
* @version DataSourceCoreServiceImpl.java, v 0.1 2022年09月23日 15:51 moji Exp $
@ -33,7 +32,6 @@ import org.springframework.util.CollectionUtils;
public class DatabaseServiceImpl implements DatabaseService {
@Override
@Cacheable(value = "database", key = "'data_source_' + #param.dataSourceId", condition = "#param.refresh == false")
public ListResult<Database> queryAll(DatabaseQueryAllParam param) {
return ListResult.of(Chat2DBContext.getMetaData().databases());
}
@ -45,9 +43,8 @@ public class DatabaseServiceImpl implements DatabaseService {
@Override
public DataResult<MetaSchema> queryDatabaseSchema(MetaDataQueryParam param) {
DataResult<MetaSchema> result = CacheManage.get("data_source_" + param.getDataSourceId(), DataResult.class);
if (result == null || param.isRefresh()) {
result = new DataResult<>();
MetaSchema ms = CacheManage.get(getDataSourceKey(param.getDataSourceId()), MetaSchema.class,
(key) -> param.isRefresh(), (key) -> {
MetaSchema metaSchema = new MetaSchema();
List<Database> databases = Chat2DBContext.getMetaData().databases();
if (!CollectionUtils.isEmpty(databases)) {
@ -68,34 +65,12 @@ public class DatabaseServiceImpl implements DatabaseService {
List<Schema> schemas = Chat2DBContext.getMetaData().schemas(null);
metaSchema.setSchemas(schemas);
}
result.setData(metaSchema);
CacheManage.put("data_source_" + param.getDataSourceId(), result);
}
return result;
return metaSchema;
});
return DataResult.of(ms);
}
public String queryDatabaseSchemaCache(MetaDataQueryParam param) {
MetaSchema metaSchema = new MetaSchema();
List<Database> databases = Chat2DBContext.getMetaData().databases();
if (!CollectionUtils.isEmpty(databases)) {
List<Schema> schemaList = Chat2DBContext.getMetaData().schemas(null);
if (databases.size() == 1) {
databases.get(0).setSchemas(schemaList);
metaSchema.setDatabases(databases);
} else {
Map<String, List<Schema>> schemaMap = schemaList.stream().collect(
Collectors.groupingBy(schema -> schema.getDatabaseName() != null ? schema.getDatabaseName() : ""));
for (Database dataBase : databases) {
dataBase.setSchemas(schemaMap.get(dataBase.getName()));
}
metaSchema.setDatabases(databases);
}
} else {
List<Schema> schemas = Chat2DBContext.getMetaData().schemas(null);
metaSchema.setSchemas(schemas);
}
return JSON.toJSONString(metaSchema);
}
@Override
public ActionResult deleteDatabase(DatabaseOperationParam param) {

33
chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java
View File

@ -6,10 +6,20 @@ import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import ai.chat2db.server.domain.api.param.*;
import ai.chat2db.server.domain.api.param.DropParam;
import ai.chat2db.server.domain.api.param.PinTableParam;
import ai.chat2db.server.domain.api.param.ShowCreateTableParam;
import ai.chat2db.server.domain.api.param.TablePageQueryParam;
import ai.chat2db.server.domain.api.param.TableQueryParam;
import ai.chat2db.server.domain.api.param.TableSelector;
import ai.chat2db.server.domain.api.service.PinService;
import ai.chat2db.server.domain.api.service.TableService;
import ai.chat2db.server.domain.core.cache.CacheManage;
import ai.chat2db.server.domain.core.converter.PinTableConverter;
import ai.chat2db.server.tools.base.wrapper.result.ActionResult;
import ai.chat2db.server.tools.base.wrapper.result.DataResult;
import ai.chat2db.server.tools.base.wrapper.result.ListResult;
import ai.chat2db.server.tools.base.wrapper.result.PageResult;
import ai.chat2db.server.tools.common.util.ContextUtils;
import ai.chat2db.spi.DBManage;
import ai.chat2db.spi.MetaData;
@ -17,12 +27,6 @@ import ai.chat2db.spi.model.Sql;
import ai.chat2db.spi.model.Table;
import ai.chat2db.spi.model.TableColumn;
import ai.chat2db.spi.model.TableIndex;
import ai.chat2db.server.tools.base.wrapper.result.ActionResult;
import ai.chat2db.server.tools.base.wrapper.result.DataResult;
import ai.chat2db.server.tools.base.wrapper.result.ListResult;
import ai.chat2db.server.tools.base.wrapper.result.PageResult;
import ai.chat2db.server.tools.common.util.EasyEnumUtils;
import ai.chat2db.spi.sql.Chat2DBContext;
import ai.chat2db.spi.util.SqlUtils;
import com.google.common.collect.Lists;
@ -30,6 +34,8 @@ import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import static ai.chat2db.server.domain.core.cache.CacheKey.getTableKey;
/**
* @author moji
* @version DataSourceCoreServiceImpl.java, v 0.1 2022年09月23日 15:51 moji Exp $
@ -93,8 +99,14 @@ public class TableServiceImpl implements TableService {
@Override
public PageResult<Table> pageQuery(TablePageQueryParam param, TableSelector selector) {
MetaData metaSchema = Chat2DBContext.getMetaData();
List<Table> list = metaSchema.tables(param.getDatabaseName(), param.getSchemaName(), param.getTableName());
list = pinTable(list,param);
String tableKey = getTableKey(param.getDataSourceId(),param.getDatabaseName(), param.getSchemaName());
List<Table> list = CacheManage.getList(tableKey, Table.class,
(key) -> param.isRefresh(), (key) ->
metaSchema.tables(param.getDatabaseName(), param.getSchemaName(), param.getTableName()));
list = pinTable(list, param);
if (CollectionUtils.isEmpty(list)) {
return PageResult.of(list, 0L, param);
}
@ -122,14 +134,13 @@ public class TableServiceImpl implements TableService {
}
for (Table table : list) {
if (table!=null && !tables.contains(table)) {
if (table != null && !tables.contains(table)) {
tables.add(table);
}
}
return tables;
}
@Override
public List<TableColumn> queryColumns(TableQueryParam param) {
MetaData metaSchema = Chat2DBContext.getMetaData();

6
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/data/source/request/DataSourceBaseRequest.java
View File

@ -27,4 +27,10 @@ public class DataSourceBaseRequest implements DataSourceBaseRequestInfo{
* 表所在空间
*/
private String schemaName;
/**
* if true, refresh the cache
*/
private boolean refresh;
}

32
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/driver/JdbcDriverController.java
View File

@ -2,16 +2,21 @@ package ai.chat2db.server.web.api.controller.driver;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import ai.chat2db.server.domain.api.service.JdbcDriverService;
import ai.chat2db.server.tools.base.wrapper.result.ActionResult;
import ai.chat2db.server.tools.base.wrapper.result.DataResult;
import ai.chat2db.server.tools.base.wrapper.result.ListResult;
import ai.chat2db.server.web.api.controller.driver.request.JdbcDriverRequest;
import ai.chat2db.spi.config.DBConfig;
import ai.chat2db.spi.util.JdbcJarUtils;
import org.apache.commons.io.FilenameUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@ -44,6 +49,7 @@ public class JdbcDriverController {
/**
* 下载驱动
*
* @param dbType
* @return
*/
@ -57,17 +63,13 @@ public class JdbcDriverController {
* 上传驱动
*
* @param multipartFiles
* @param jdbcDriverClass
* @return
*/
@PostMapping("/upload")
public ActionResult upload(@RequestParam MultipartFile[] multipartFiles, @RequestParam String jdbcDriverClass,
@RequestParam String dbType) {
StringBuilder stringBuilder = new StringBuilder();
public ListResult<String> upload(@RequestParam MultipartFile[] multipartFiles) {
List<String> list = new ArrayList<>();
for (int i = 0; i < multipartFiles.length; i++) {
if (i > 0) {
stringBuilder.append(",");
}
MultipartFile multipartFile = multipartFiles[i];
String originalFilename = FilenameUtils.getName(multipartFile.getOriginalFilename());
String location = JdbcJarUtils.PATH + originalFilename;
@ -76,10 +78,22 @@ public class JdbcDriverController {
} catch (IOException e) {
throw new RuntimeException(e);
}
stringBuilder.append(originalFilename);
list.add(originalFilename);
}
return ListResult.of(list);
}
return jdbcDriverService.upload(dbType, jdbcDriverClass, stringBuilder.toString());
/**
* save
*
* @param request
* @return
*/
@PostMapping("/save")
public ActionResult save(@RequestBody JdbcDriverRequest request) {
return jdbcDriverService.upload(request.getDbType(), request.getJdbcDriverClass(),
String.join(",", request.getJdbcDriver()));
}
///**

13
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/driver/request/JdbcDriverRequest.java Normal file
View File

@ -0,0 +1,13 @@
package ai.chat2db.server.web.api.controller.driver.request;
import java.util.List;
import lombok.Data;
@Data
public class JdbcDriverRequest {
String jdbcDriverClass;
String dbType;
List<String> jdbcDriver;
}

3
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDdlController.java
View File

@ -100,7 +100,8 @@ public class RdbDdlController {
*/
@GetMapping("/database_schema_list")
public DataResult<MetaSchemaVO> databaseSchemaList(@Valid DataSourceBaseRequest request) {
MetaDataQueryParam queryParam = MetaDataQueryParam.builder().dataSourceId(request.getDataSourceId()).build();
MetaDataQueryParam queryParam = MetaDataQueryParam.builder().dataSourceId(request.getDataSourceId()).refresh(
request.isRefresh()).build();
DataResult<MetaSchema> result = databaseService.queryDatabaseSchema(queryParam);
MetaSchemaVO schemaDto2vo = rdbWebConverter.metaSchemaDto2vo(result.getData());
return DataResult.of(schemaDto2vo);

5
chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableBriefQueryRequest.java
View File

@ -38,4 +38,9 @@ public class TableBriefQueryRequest extends PageQueryRequest implements DataSour
* 模糊搜索词
*/
private String searchKey;
/**
* if true, refresh the cache
*/
private boolean refresh;
}

15
chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java
View File

@ -152,7 +152,6 @@ public class SQLExecutor {
return execute(sql, getConnection());
}
/**
* 获取所有的数据库
*
@ -181,7 +180,7 @@ public class SQLExecutor {
*/
public List<Map<String, String>> schemas(String databaseName, String schemaName) {
List<Map<String, String>> schemaList = Lists.newArrayList();
if(StringUtils.isEmpty(databaseName) && StringUtils.isEmpty(schemaName)){
if (StringUtils.isEmpty(databaseName) && StringUtils.isEmpty(schemaName)) {
try (ResultSet resultSet = getConnection().getMetaData().getSchemas()) {
if (resultSet != null) {
while (resultSet.next()) {
@ -192,7 +191,7 @@ public class SQLExecutor {
}
}
} catch (SQLException e) {
throw new RuntimeException("Get schemas error",e);
throw new RuntimeException("Get schemas error", e);
}
return schemaList;
}
@ -206,7 +205,7 @@ public class SQLExecutor {
}
}
} catch (SQLException e) {
throw new RuntimeException("Get schemas error",e);
throw new RuntimeException("Get schemas error", e);
}
return schemaList;
}
@ -222,11 +221,16 @@ public class SQLExecutor {
*/
public List<Table> tables(String databaseName, String schemaName, String tableName, String types[]) {
List<Table> tables = Lists.newArrayList();
int n = 0;
try (ResultSet resultSet = getConnection().getMetaData().getTables(databaseName, schemaName, tableName,
types)) {
if (resultSet != null) {
while (resultSet.next()) {
n++;
tables.add(buildTable(resultSet));
if (n >= 5000) {// 最多只取5000条
break;
}
}
}
} catch (SQLException e) {
@ -269,7 +273,8 @@ public class SQLExecutor {
*/
public List<TableIndex> indexes(String databaseName, String schemaName, String tableName) {
List<TableIndex> tableIndices = Lists.newArrayList();
try (ResultSet resultSet = getConnection().getMetaData().getIndexInfo(databaseName, schemaName, tableName, false,
try (ResultSet resultSet = getConnection().getMetaData().getIndexInfo(databaseName, schemaName, tableName,
false,
false)) {
List<TableIndexColumn> tableIndexColumns = Lists.newArrayList();