diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java index 0d30804c..8456fce4 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java @@ -1,8 +1,293 @@ package ai.chat2db.plugin.clickhouse; +import ai.chat2db.plugin.clickhouse.builder.ClickHouseSqlBuilder; +import ai.chat2db.plugin.clickhouse.type.ClickHouseColumnTypeEnum; +import ai.chat2db.plugin.clickhouse.type.ClickHouseEngineTypeEnum; +import ai.chat2db.plugin.clickhouse.type.ClickHouseIndexTypeEnum; import ai.chat2db.spi.MetaData; +import ai.chat2db.spi.SqlBuilder; import ai.chat2db.spi.jdbc.DefaultMetaService; +import ai.chat2db.spi.model.*; +import ai.chat2db.spi.sql.SQLExecutor; +import jakarta.validation.constraints.NotEmpty; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.stream.Collectors; + +import static ai.chat2db.spi.util.SortUtils.sortDatabase; public class ClickHouseMetaData extends DefaultMetaService implements MetaData { + + private static String ROUTINES_SQL + = + "SELECT SPECIFIC_NAME, ROUTINE_COMMENT, ROUTINE_DEFINITION FROM information_schema.routines WHERE " + + "routine_type = '%s' AND ROUTINE_SCHEMA ='%s' AND " + + "routine_name = '%s';"; + private static String TRIGGER_SQL + = "SELECT TRIGGER_NAME,EVENT_MANIPULATION, ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS where " + + "TRIGGER_SCHEMA = '%s' AND TRIGGER_NAME = '%s';"; + private static String TRIGGER_SQL_LIST + = "SELECT TRIGGER_NAME FROM INFORMATION_SCHEMA.TRIGGERS where TRIGGER_SCHEMA = '%s';"; + private static String SELECT_TABLE_COLUMNS = "SELECT * FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' order by ORDINAL_POSITION"; + private static String VIEW_SQL + = "SELECT TABLE_SCHEMA AS DatabaseName, TABLE_NAME AS ViewName, VIEW_DEFINITION AS definition, CHECK_OPTION, " + + "IS_UPDATABLE FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';"; + private List systemDatabases = Arrays.asList("information_schema", "system"); + + public static String format(String tableName) { + return "`" + tableName + "`"; + } + + @Override + public List databases(Connection connection) { + List list = SQLExecutor.getInstance().execute(connection, "SELECT name FROM system.databases;;", resultSet -> { + List databases = new ArrayList<>(); + try { + while (resultSet.next()) { + String dbName = resultSet.getString("name"); + Database database = new Database(); + database.setName(dbName); + databases.add(database); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + return databases; + }); + return sortDatabase(list, systemDatabases, connection); + } + + @Override + public String tableDDL(Connection connection, @NotEmpty String databaseName, String schemaName, + @NotEmpty String tableName) { + String sql = "SHOW CREATE TABLE " + format(databaseName) + "." + + format(tableName); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + if (resultSet.next()) { + return resultSet.getString("Create Table"); + } + return null; + }); + } + + @Override + public Function function(Connection connection, @NotEmpty String databaseName, String schemaName, + String functionName) { + + String sql = String.format(ROUTINES_SQL, "FUNCTION", databaseName, functionName); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + Function function = new Function(); + function.setDatabaseName(databaseName); + function.setSchemaName(schemaName); + function.setFunctionName(functionName); + if (resultSet.next()) { + function.setSpecificName(resultSet.getString("SPECIFIC_NAME")); + function.setRemarks(resultSet.getString("ROUTINE_COMMENT")); + function.setFunctionBody(resultSet.getString("ROUTINE_DEFINITION")); + } + return function; + }); + + } + + @Override + public List triggers(Connection connection, String databaseName, String schemaName) { + List triggers = new ArrayList<>(); + String sql = String.format(TRIGGER_SQL_LIST, databaseName); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + while (resultSet.next()) { + Trigger trigger = new Trigger(); + trigger.setTriggerName(resultSet.getString("TRIGGER_NAME")); + trigger.setSchemaName(schemaName); + trigger.setDatabaseName(databaseName); + triggers.add(trigger); + } + return triggers; + }); + } + + @Override + public Trigger trigger(Connection connection, @NotEmpty String databaseName, String schemaName, + String triggerName) { + + String sql = String.format(TRIGGER_SQL, databaseName, triggerName); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + Trigger trigger = new Trigger(); + trigger.setDatabaseName(databaseName); + trigger.setSchemaName(schemaName); + trigger.setTriggerName(triggerName); + if (resultSet.next()) { + trigger.setTriggerBody(resultSet.getString("ACTION_STATEMENT")); + } + return trigger; + }); + } + + @Override + public Procedure procedure(Connection connection, @NotEmpty String databaseName, String schemaName, + String procedureName) { + String sql = String.format(ROUTINES_SQL, "PROCEDURE", databaseName, procedureName); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + Procedure procedure = new Procedure(); + procedure.setDatabaseName(databaseName); + procedure.setSchemaName(schemaName); + procedure.setProcedureName(procedureName); + if (resultSet.next()) { + procedure.setSpecificName(resultSet.getString("SPECIFIC_NAME")); + procedure.setRemarks(resultSet.getString("ROUTINE_COMMENT")); + procedure.setProcedureBody(resultSet.getString("ROUTINE_DEFINITION")); + } + return procedure; + }); + } + + @Override + public List columns(Connection connection, String databaseName, String schemaName, String tableName) { + String sql = String.format(SELECT_TABLE_COLUMNS, databaseName, tableName); + List tableColumns = new ArrayList<>(); + + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + while (resultSet.next()) { + TableColumn column = new TableColumn(); + column.setDatabaseName(databaseName); + column.setTableName(tableName); + column.setOldName(resultSet.getString("COLUMN_NAME")); + column.setName(resultSet.getString("COLUMN_NAME")); + String dataType = resultSet.getString("DATA_TYPE"); + if (dataType.startsWith("Nullable(")) { + dataType = dataType.substring(9, dataType.length() - 1); + } + column.setColumnType(dataType); + column.setDefaultValue(resultSet.getString("COLUMN_DEFAULT")); + column.setAutoIncrement(resultSet.getString("EXTRA").contains("auto_increment")); + column.setComment(resultSet.getString("COLUMN_COMMENT")); + column.setNullable("YES".equalsIgnoreCase(resultSet.getString("IS_NULLABLE")) ? 1 : 0); + column.setOrdinalPosition(resultSet.getInt("ORDINAL_POSITION")); + column.setDecimalDigits(resultSet.getInt("NUMERIC_SCALE")); + column.setCharSetName(resultSet.getString("CHARACTER_SET_NAME")); + column.setCollationName(resultSet.getString("COLLATION_NAME")); + setColumnSize(column, resultSet.getString("COLUMN_TYPE")); + tableColumns.add(column); + } + return tableColumns; + }); + } + + private void setColumnSize(TableColumn column, String columnType) { + try { + if (columnType.contains("(")) { + String size = columnType.substring(columnType.indexOf("(") + 1, columnType.indexOf(")")); + if ("SET".equalsIgnoreCase(column.getColumnType()) || "ENUM".equalsIgnoreCase(column.getColumnType())) { + column.setValue(size); + } else { + if (size.contains(",")) { + String[] sizes = size.split(","); + if (StringUtils.isNotBlank(sizes[0])) { + column.setColumnSize(Integer.parseInt(sizes[0])); + } + if (StringUtils.isNotBlank(sizes[1])) { + column.setDecimalDigits(Integer.parseInt(sizes[1])); + } + } else { + column.setColumnSize(Integer.parseInt(size)); + } + } + } + } catch (Exception e) { + } + } + + @Override + public Table view(Connection connection, String databaseName, String schemaName, String viewName) { + String sql = String.format(VIEW_SQL, databaseName, viewName); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + Table table = new Table(); + table.setDatabaseName(databaseName); + table.setSchemaName(schemaName); + table.setName(viewName); + if (resultSet.next()) { + table.setDdl(resultSet.getString("definition")); + } + return table; + }); + } + + + @Override + public List indexes(Connection connection, String databaseName, String schemaName, String tableName) { + StringBuilder queryBuf = new StringBuilder("SHOW INDEX FROM "); + queryBuf.append("`").append(tableName).append("`"); + queryBuf.append(" FROM "); + queryBuf.append("`").append(databaseName).append("`"); + return SQLExecutor.getInstance().execute(connection, queryBuf.toString(), resultSet -> { + LinkedHashMap map = new LinkedHashMap(); + while (resultSet.next()) { + String keyName = resultSet.getString("Key_name"); + + TableIndex index = new TableIndex(); + index.setDatabaseName(databaseName); + index.setSchemaName(schemaName); + index.setTableName(tableName); + index.setName(keyName); + index.setUnique(!resultSet.getBoolean("Non_unique")); + index.setType(resultSet.getString("Index_type")); +// index.setComment(resultSet.getString("Index_comment")); + List tableIndexColumns = new ArrayList<>(); + tableIndexColumns.addAll(getTableIndexColumn(resultSet)); + index.setColumnList(tableIndexColumns); + if ("PRIMARY".equalsIgnoreCase(keyName)) { + index.setType(ClickHouseIndexTypeEnum.PRIMARY.getName()); + } + map.put(keyName, index); + } + return map.values().stream().collect(Collectors.toList()); + }); + + } + + private List getTableIndexColumn(ResultSet resultSet) throws SQLException { + List tableIndexColumns = new ArrayList<>(); + String name = StringUtils.isBlank(resultSet.getString("column_name")) ? resultSet.getString("expression") : resultSet.getString("column_name"); + if (StringUtils.isNotBlank(name)) { + String[] split = name.split(","); + for (String columName : split) { + TableIndexColumn tableIndexColumn = new TableIndexColumn(); + tableIndexColumn.setColumnName(columName); + tableIndexColumn.setOrdinalPosition(resultSet.getShort("seq_in_index")); + tableIndexColumn.setCollation(resultSet.getString("collation")); + tableIndexColumn.setCardinality(resultSet.getLong("cardinality")); + tableIndexColumn.setSubPart(resultSet.getLong("sub_part")); + tableIndexColumns.add(tableIndexColumn); + } + } + return tableIndexColumns; + } + + @Override + public SqlBuilder getSqlBuilder() { + return new ClickHouseSqlBuilder(); + } + + @Override + public TableMeta getTableMeta(String databaseName, String schemaName, String tableName) { + return TableMeta.builder() + .columnTypes(ClickHouseColumnTypeEnum.getTypes()) + .engineTypes(ClickHouseEngineTypeEnum.getTypes()) + .indexTypes(ClickHouseIndexTypeEnum.getIndexTypes()) + .build(); + } + + @Override + public String getMetaDataName(String... names) { + return Arrays.stream(names).filter(name -> StringUtils.isNotBlank(name)).map(name -> "`" + name + "`").collect(Collectors.joining(".")); + } + } diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java new file mode 100644 index 00000000..41140ef5 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java @@ -0,0 +1,137 @@ +package ai.chat2db.plugin.clickhouse.builder; + +import ai.chat2db.plugin.clickhouse.type.ClickHouseColumnTypeEnum; +import ai.chat2db.plugin.clickhouse.type.ClickHouseIndexTypeEnum; +import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.jdbc.DefaultSqlBuilder; +import ai.chat2db.spi.model.Database; +import ai.chat2db.spi.model.Table; +import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.model.TableIndex; +import org.apache.commons.lang3.StringUtils; + + +public class ClickHouseSqlBuilder extends DefaultSqlBuilder implements SqlBuilder { + @Override + public String buildCreateTableSql(Table table) { + StringBuilder script = new StringBuilder(); + script.append("CREATE TABLE "); + if (StringUtils.isNotBlank(table.getDatabaseName())) { + script.append("`").append(table.getDatabaseName()).append("`").append("."); + } + script.append("`").append(table.getName()).append("`").append(" (").append("\n"); + + // append column + for (TableColumn column : table.getColumnList()) { + if (StringUtils.isBlank(column.getName()) || StringUtils.isBlank(column.getColumnType())) { + continue; + } + ClickHouseColumnTypeEnum typeEnum = ClickHouseColumnTypeEnum.getByType(column.getColumnType()); + script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); + } + + // append index + for (TableIndex tableIndex : table.getIndexList()) { + if (StringUtils.isBlank(tableIndex.getName()) || StringUtils.isBlank(tableIndex.getType())) { + continue; + } + ClickHouseIndexTypeEnum mysqlIndexTypeEnum = ClickHouseIndexTypeEnum.getByType(tableIndex.getType()); + if (!ClickHouseIndexTypeEnum.PRIMARY.equals(mysqlIndexTypeEnum) ) { + script.append("\t").append("").append(mysqlIndexTypeEnum.buildIndexScript(tableIndex)).append(",\n"); + } + } + + script = new StringBuilder(script.substring(0, script.length() - 2)); + script.append("\n)"); + + + if (StringUtils.isNotBlank(table.getEngine())) { + script.append(" ENGINE=").append(table.getEngine()).append("\n"); + } + // append primary key + for (TableIndex tableIndex : table.getIndexList()) { + if (StringUtils.isBlank(tableIndex.getName()) || StringUtils.isBlank(tableIndex.getType())) { + continue; + } + ClickHouseIndexTypeEnum mysqlIndexTypeEnum = ClickHouseIndexTypeEnum.getByType(tableIndex.getType()); + if (ClickHouseIndexTypeEnum.PRIMARY.equals(mysqlIndexTypeEnum) ) { + script.append("\t").append("").append(mysqlIndexTypeEnum.buildIndexScript(tableIndex)).append("\n"); + } + } + + if (StringUtils.isNotBlank(table.getComment())) { + script.append(" COMMENT '").append(table.getComment()).append("'"); + } + + script.append(";"); + + return script.toString(); + } + + @Override + public String buildModifyTaleSql(Table oldTable, Table newTable) { + StringBuilder script = new StringBuilder(); + script.append("ALTER TABLE "); + if (StringUtils.isNotBlank(oldTable.getDatabaseName())) { + script.append("`").append(oldTable.getDatabaseName()).append("`").append("."); + } + script.append("`").append(oldTable.getName()).append("`").append("\n"); + + if (!StringUtils.equalsIgnoreCase(oldTable.getComment(), newTable.getComment())) { + script.append("\t").append("MODIFY COMMENT").append("'").append(newTable.getComment()).append("'").append(",\n"); + } + + // append modify column + for (TableColumn tableColumn : newTable.getColumnList()) { + if (StringUtils.isNotBlank(tableColumn.getEditStatus()) && StringUtils.isNotBlank(tableColumn.getColumnType()) && StringUtils.isNotBlank(tableColumn.getName())) { + ClickHouseColumnTypeEnum typeEnum = ClickHouseColumnTypeEnum.getByType(tableColumn.getColumnType()); + script.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(",\n"); + } + } + + // append modify index + for (TableIndex tableIndex : newTable.getIndexList()) { + if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { + ClickHouseIndexTypeEnum mysqlIndexTypeEnum = ClickHouseIndexTypeEnum + .getByType(tableIndex.getType()); + script.append("\t").append(mysqlIndexTypeEnum.buildModifyIndex(tableIndex)).append(",\n"); + } + } + + if (script.length() > 2) { + script = new StringBuilder(script.substring(0, script.length() - 2)); + script.append(";"); + } + + return script.toString(); + } + + + @Override + public String pageLimit(String sql, int offset, int pageNo, int pageSize) { + StringBuilder sqlBuilder = new StringBuilder(sql.length() + 14); + sqlBuilder.append(sql); + if (offset == 0) { + sqlBuilder.append("\n LIMIT "); + sqlBuilder.append(pageSize); + } else { + sqlBuilder.append("\n LIMIT "); + sqlBuilder.append(offset); + sqlBuilder.append(","); + sqlBuilder.append(pageSize); + } + return sqlBuilder.toString(); + } + + + @Override + public String buildCreateDatabaseSql(Database database) { + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("CREATE DATABASE `" + database.getName() + "`"); + if(StringUtils.isNotBlank(database.getComment())){ + sqlBuilder.append(";ALTER DATABASE ").append(database.getName()).append(" COMMENT '").append(database.getComment()).append("';"); + } + return sqlBuilder.toString(); + } + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/clickhouse.json b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/clickhouse.json index c99541f5..d6c98c73 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/clickhouse.json +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/clickhouse.json @@ -1,7 +1,7 @@ { "dbType": "CLICKHOUSE", - "supportDatabase": false, - "supportSchema": true, + "supportDatabase": true, + "supportSchema":false, "driverConfigList": [ { "url": "jdbc:clickhouse://localhost:8123/", diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseColumnTypeEnum.java new file mode 100644 index 00000000..d53c90cc --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseColumnTypeEnum.java @@ -0,0 +1,221 @@ +package ai.chat2db.plugin.clickhouse.type; + +import ai.chat2db.spi.ColumnBuilder; +import ai.chat2db.spi.enums.EditStatus; +import ai.chat2db.spi.model.ColumnType; +import ai.chat2db.spi.model.TableColumn; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public enum ClickHouseColumnTypeEnum implements ColumnBuilder { + + String("String", false, false, true, false, false, false, true, true, false, false), + Int8("Int8", false, false, true, false, false, false, true, true, false, false), + Int16("Int16", false, false, true, false, false, false, true, true, false, false), + Int32("Int32", false, false, true, false, false, false, true, true, false, false), + Int64("Int64", false, false, true, false, false, false, true, true, false, false), + Int128("Int128", false, false, true, false, false, false, true, true, false, false), + Int256("Int256", false, false, true, false, false, false, true, true, false, false), + UInt8("UInt8", false, false, true, false, false, false, true, true, false, false), + UInt16("UInt16", false, false, true, false, false, false, true, true, false, false), + UInt32("UInt32", false, false, true, false, false, false, true, true, false, false), + UInt64("UInt64", false, false, true, false, false, false, true, true, false, false), + UInt128("UInt128", false, false, true, false, false, false, true, true, false, false), + UInt256("UInt256", false, false, true, false, false, false, true, true, false, false), + Float32("Float32", false, false, true, false, false, false, true, true, false, false), + Float64("Float64", false, false, true, false, false, false, true, true, false, false), + Decimal("Decimal", true, true, true, false, false, false, true, true, false, false), + Boolean("Boolean", false, false, true, false, false, false, true, true, false, false), + FixedString("FixedString", false, false, true, false, false, false, true, true, false, false), + UUID("UUID", false, false, true, false, false, false, true, true, false, false), + Date("Date", false, false, true, false, false, false, true, true, false, false), + DATE32("DATE32", false, false, true, false, false, false, true, true, false, false), + DateTime("DateTime", false, false, true, false, false, false, true, true, false, false), + DateTime64("DateTime64", false, false, true, false, false, false, true, true, false, false), + Enum8("Enum8", false, false, true, false, false, false, true, true, false, false), + Enum16("Enum16", false, false, true, false, false, false, true, true, false, false), + Array("Array", false, false, false, false, false, false, true, true, false, false), + JSON("JSON", false, false, true, false, false, false, true, true, false, false), + Nested("Nested", false, false, true, false, false, false, true, true, false, false), + Map("Map", true, true, true, false, false, false, true, true, false, false), + IPv4("IPv4", false, false, true, false, false, false, true, true, false, false), + IPv6("IPv6", false, false, true, false, false, false, true, true, false, false), + Point("Point", false, false, true, false, false, false, true, true, false, false), + Ring("Ring", false, false, true, false, false, false, true, true, false, false), + Polygon("Polygon", false, false, true, false, false, false, true, true, false, false), + MultiPolygon("MultiPolygon", false, false, true, false, false, false, true, true, false, false), + AggregateFunction("AggregateFunction", true, true, true, false, false, false, true, true, false, false), + SimpleAggregateFunction("SimpleAggregateFunction", true, true, true, false, false, false, true, true, false, false), + ; + private static Map COLUMN_TYPE_MAP = Maps.newHashMap(); + + static { + for (ClickHouseColumnTypeEnum value : ClickHouseColumnTypeEnum.values()) { + COLUMN_TYPE_MAP.put(value.getColumnType().getTypeName(), value); + } + } + + private ColumnType columnType; + + + ClickHouseColumnTypeEnum(String dataTypeName, boolean supportLength, boolean supportScale, boolean supportNullable, boolean supportAutoIncrement, boolean supportCharset, boolean supportCollation, boolean supportComments, boolean supportDefaultValue, boolean supportExtent, boolean supportValue) { + this.columnType = new ColumnType(dataTypeName, supportLength, supportScale, supportNullable, supportAutoIncrement, supportCharset, supportCollation, supportComments, supportDefaultValue, supportExtent, supportValue, false); + } + + public static ClickHouseColumnTypeEnum getByType(String dataType) { + return COLUMN_TYPE_MAP.get(dataType); + } + + public static List getTypes() { + return Arrays.stream(ClickHouseColumnTypeEnum.values()).map(columnTypeEnum -> + columnTypeEnum.getColumnType() + ).toList(); + } + + public ColumnType getColumnType() { + return columnType; + } + + @Override + public String buildCreateColumnSql(TableColumn column) { + ClickHouseColumnTypeEnum type = COLUMN_TYPE_MAP.get(column.getColumnType()); + if (type == null) { + return ""; + } + StringBuilder script = new StringBuilder(); + + script.append("`").append(column.getName()).append("`").append(" "); + + script.append(buildNullableAndDataType(column, type)).append(" "); + + script.append(buildDefaultValue(column, type)).append(" "); + + script.append(buildComment(column, type)).append(" "); + + return script.toString(); + } + + @Override + public String buildModifyColumn(TableColumn tableColumn) { + + if (EditStatus.DELETE.name().equals(tableColumn.getEditStatus())) { + return StringUtils.join("DROP COLUMN `", tableColumn.getName() + "`"); + } + if (EditStatus.ADD.name().equals(tableColumn.getEditStatus())) { + return StringUtils.join("ADD COLUMN ", buildCreateColumnSql(tableColumn)); + } + if (EditStatus.MODIFY.name().equals(tableColumn.getEditStatus())) { + String modifyColumn = ""; + if (!StringUtils.equalsIgnoreCase(tableColumn.getOldName(), tableColumn.getName())) { + modifyColumn = StringUtils.join("RENAME COLUMN `", tableColumn.getOldName(), "` TO `", tableColumn.getName(), + "`, ", buildCreateColumnSql(tableColumn)); + } + return StringUtils.join(modifyColumn, "MODIFY COLUMN ", buildCreateColumnSql(tableColumn)); + } + return ""; + } + + private String buildComment(TableColumn column, ClickHouseColumnTypeEnum type) { + if (!type.columnType.isSupportComments() || StringUtils.isEmpty(column.getComment())) { + return ""; + } + return StringUtils.join("COMMENT '", column.getComment(), "'"); + } + + private String buildDefaultValue(TableColumn column, ClickHouseColumnTypeEnum type) { + if (!type.getColumnType().isSupportDefaultValue() || StringUtils.isEmpty(column.getDefaultValue())) { + return ""; + } + + if ("EMPTY_STRING".equalsIgnoreCase(column.getDefaultValue().trim())) { + return StringUtils.join("DEFAULT ''"); + } + + if ("NULL".equalsIgnoreCase(column.getDefaultValue().trim())) { + return StringUtils.join("DEFAULT NULL"); + } + + if (Arrays.asList(Enum8,Enum16).contains(type)) { + return StringUtils.join("DEFAULT '", column.getDefaultValue(), "'"); + } + + if (Arrays.asList(Date).contains(type)) { + return StringUtils.join("DEFAULT '", column.getDefaultValue(), "'"); + } + + if (Arrays.asList(DateTime).contains(type)) { + if ("CURRENT_TIMESTAMP".equalsIgnoreCase(column.getDefaultValue().trim())) { + return StringUtils.join("DEFAULT ", column.getDefaultValue()); + } + return StringUtils.join("DEFAULT '", column.getDefaultValue(), "'"); + } + + return StringUtils.join("DEFAULT ", column.getDefaultValue()); + } + + private String buildNullableAndDataType(TableColumn column, ClickHouseColumnTypeEnum type) { + StringBuilder script = new StringBuilder(); + script.append(buildDataType(column, type)); + + if (!type.getColumnType().isSupportNullable()) { + return script.toString(); + } + if (column.getNullable() != null && 1 == column.getNullable()) { + return "Nullable("+script.append(")").toString(); + } else { + return script.toString(); + } + } + + private String buildDataType(TableColumn column, ClickHouseColumnTypeEnum type) { + String columnType = type.columnType.getTypeName(); + if (Arrays.asList(FixedString).contains(type)) { + return StringUtils.join(columnType, "(", column.getColumnSize(), ")"); + } + + + if (Arrays.asList(Decimal).contains(type)) { + if (column.getColumnSize() == null || column.getDecimalDigits() == null) { + return columnType; + } + if (column.getColumnSize() != null && column.getDecimalDigits() == null) { + return StringUtils.join(columnType, "(", column.getColumnSize() + ")"); + } + if (column.getColumnSize() != null && column.getDecimalDigits() != null) { + return StringUtils.join(columnType, "(", column.getColumnSize() + "," + column.getDecimalDigits() + ")"); + } + } + + return columnType; + + + } + + public String buildColumn(TableColumn column) { + ClickHouseColumnTypeEnum type = COLUMN_TYPE_MAP.get(column.getColumnType()); + if (type == null) { + return ""; + } + StringBuilder script = new StringBuilder(); + + script.append("`").append(column.getName()).append("`").append(" "); + script.append(buildDataType(column, type)).append(" "); + if (StringUtils.isNoneBlank(column.getComment())) { + script.append("COMMENT").append(" ").append("'").append(column.getComment()).append("'").append(" "); + } + return script.toString(); + } + + private String unsignedDataType(String dataTypeName, String middle) { + String[] split = dataTypeName.split(" "); + if (split.length == 2) { + return StringUtils.join(split[0], middle, split[1]); + } + return StringUtils.join(dataTypeName, middle); + } + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseEngineTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseEngineTypeEnum.java new file mode 100644 index 00000000..902b2154 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseEngineTypeEnum.java @@ -0,0 +1,101 @@ +package ai.chat2db.plugin.clickhouse.type; + +import ai.chat2db.spi.model.EngineType; +import com.google.common.collect.Maps; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public enum ClickHouseEngineTypeEnum { + AzureBlobStorage("AzureBlobStorage",false,true,false,false,true,false,false,false ), + KeeperMap("KeeperMap",false,true,false,false,false,true,false,false ), + SQLite("SQLite",false,false,false,false,false,false,false,false ), + ExternalDistributed("ExternalDistributed",false,false,false,false,false,false,false,false ), + PostgreSQL("PostgreSQL",false,false,false,false,false,false,false,false ), + NATS("NATS",false,false,false,false,true,false,false,false ), + RabbitMQ("RabbitMQ",false,false,false,false,true,false,false,false ), + Kafka("Kafka",false,false,false,false,true,false,false,false ), + MongoDB("MongoDB",false,false,false,false,false,false,false,false ), + FileLog("FileLog",false,false,false,false,true,false,false,false ), + Dictionary("Dictionary",false,false,false,false,false,false,false,false ), + MySQL("MySQL",false,false,false,false,true,false,false,false ), + S3Queue("S3Queue",false,false,false,false,true,false,false,false ), + HDFS("HDFS",false,true,false,false,false,false,false,false ), + MaterializedPostgreSQL("MaterializedPostgreSQL",false,true,false,false,true,false,false,false ), + S3("S3",false,true,false,false,true,false,false,false ), + FuzzJSON("FuzzJSON",false,false,false,false,false,false,false,false ), + OSS("OSS",false,true,false,false,true,false,false,false ), + WindowView("WindowView",false,false,false,false,false,false,false,false ), + Distributed("Distributed",false,false,false,false,true,true,false,false ), + ReplicatedSummingMergeTree("ReplicatedSummingMergeTree",true,true,true,true,true,true,true,true ), + ExecutablePool("ExecutablePool",false,false,false,false,true,false,false,false ), + COSN("COSN",false,true,false,false,true,false,false,false ), + Iceberg("Iceberg",false,false,false,false,false,false,false,false ), + MaterializedView("MaterializedView",false,false,false,false,false,false,false,false ), + View("View",false,false,false,false,false,false,false,false ), + JDBC("JDBC",false,false,false,false,false,false,false,false ), + Join("Join",false,false,false,false,true,false,false,false ), + Executable("Executable",false,false,false,false,true,false,false,false ), + Set("Set",false,false,false,false,true,false,false,false ), + Redis("Redis",false,true,false,false,false,true,false,false ), + GenerateRandom("GenerateRandom",false,false,false,false,false,false,false,false ), + LiveView("LiveView",false,false,false,false,false,false,false,false ), + MergeTree("MergeTree",true,true,true,false,true,true,true,false ), + ReplicatedReplacingMergeTree("ReplicatedReplacingMergeTree",true,true,true,true,true,true,true,true ), + Memory("Memory",false,false,false,false,true,true,false,false ), + Buffer("Buffer",false,false,false,false,false,true,false,false ), + URL("URL",false,false,false,false,true,false,false,false ), + ReplicatedVersionedCollapsingMergeTree("ReplicatedVersionedCollapsingMergeTree",true,true,true,true,true,true,true,true ), + VersionedCollapsingMergeTree("VersionedCollapsingMergeTree",true,true,true,false,true,true,true,false ), + Hive("Hive",false,true,false,false,true,false,false,false ), + ReplacingMergeTree("ReplacingMergeTree",true,true,true,false,true,true,true,false ), + ReplicatedAggregatingMergeTree("ReplicatedAggregatingMergeTree",true,true,true,true,true,true,true,true ), + ReplicatedMergeTree("ReplicatedMergeTree",true,true,true,true,true,true,true,true ), + DeltaLake("DeltaLake",false,false,false,false,false,false,false,false ), + EmbeddedRocksDB("EmbeddedRocksDB",true,true,false,false,false,true,false,false ), + ReplicatedCollapsingMergeTree("ReplicatedCollapsingMergeTree",true,true,true,true,true,true,true,true ), + File("File",false,false,false,false,true,false,false,false ), + TinyLog("TinyLog",false,false,false,false,true,false,false,false ), + ReplicatedGraphiteMergeTree("ReplicatedGraphiteMergeTree",true,true,true,true,true,true,true,true ), + SummingMergeTree("SummingMergeTree",true,true,true,false,true,true,true,false ), + Hudi("Hudi",false,false,false,false,false,false,false,false ), + GraphiteMergeTree("GraphiteMergeTree",true,true,true,false,true,true,true,false ), + CollapsingMergeTree("CollapsingMergeTree",true,true,true,false,true,true,true,false ), + Merge("Merge",false,false,false,false,false,false,false,false ), + AggregatingMergeTree("AggregatingMergeTree",true,true,true,false,true,true,true,false ), + ODBC("ODBC",false,false,false,false,false,false,false,false ), + Null("Null",false,false,false,false,false,true,false,false ), + StripeLog("StripeLog",false,false,false,false,true,false,false,false ), + Log("Log",false,false,false,false,true,false,false,false ), + + ; + private static Map ENGINE_TYPE_MAP = Maps.newHashMap(); + + static { + for (ClickHouseEngineTypeEnum value : ClickHouseEngineTypeEnum.values()) { + ENGINE_TYPE_MAP.put(value.getEngineType().getName(), value); + } + } + + private EngineType engineType; + + + ClickHouseEngineTypeEnum(String name, boolean supportTTL, boolean supportSortOrder, boolean supportSkippingIndices, boolean supportDeduplication, boolean supportSettings, boolean supportParallelInsert, boolean supportProjections, boolean supportReplication) { + this.engineType = new EngineType(name, supportTTL, supportSortOrder, supportSkippingIndices, supportDeduplication, supportSettings, supportParallelInsert, supportProjections, supportReplication); + } + + public static ClickHouseEngineTypeEnum getByType(String dataType) { + return ENGINE_TYPE_MAP.get(dataType.toUpperCase()); + } + + public static List getTypes() { + return Arrays.stream(ClickHouseEngineTypeEnum.values()).map(engineTypeEnum -> + engineTypeEnum.getEngineType() + ).toList(); + } + + public EngineType getEngineType() { + return engineType; + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseIndexTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseIndexTypeEnum.java new file mode 100644 index 00000000..7321ab46 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseIndexTypeEnum.java @@ -0,0 +1,123 @@ +package ai.chat2db.plugin.clickhouse.type; + +import ai.chat2db.spi.enums.EditStatus; +import ai.chat2db.spi.model.IndexType; +import ai.chat2db.spi.model.TableIndex; +import ai.chat2db.spi.model.TableIndexColumn; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; + +public enum ClickHouseIndexTypeEnum { + + PRIMARY("Primary", "PRIMARY KEY"), + MINMAX("MINMAX", "INDEX"), + SET("SET", "INDEX"), + BLOOM_FILTER("BLOOM_FILTER", "INDEX"), + TOKENBF_V1("TOKENBF_V1", "INDEX"), + NGRAMBF_V1("NGRAMBF_V1", "INDEX"), + INVERTED("INVERTED", "INDEX"), + HYPOTHESIS("HYPOTHESIS", "INDEX"), + ANNOY("ANNOY", "INDEX"), + USEARCH("USEARCH", "INDEX"), + + ; + + private String name; + private String keyword; + private IndexType indexType; + + ClickHouseIndexTypeEnum(String name, String keyword) { + this.name = name; + this.keyword = keyword; + this.indexType = new IndexType(name); + } + + public static ClickHouseIndexTypeEnum getByType(String type) { + for (ClickHouseIndexTypeEnum value : ClickHouseIndexTypeEnum.values()) { + if (value.name.equalsIgnoreCase(type)) { + return value; + } + } + return null; + } + + public static List getIndexTypes() { + return Arrays.asList(ClickHouseIndexTypeEnum.values()).stream().map(ClickHouseIndexTypeEnum::getIndexType).collect(java.util.stream.Collectors.toList()); + } + + public String getName() { + return name; + } + + public String getKeyword() { + return keyword; + } + + public IndexType getIndexType() { + return indexType; + } + + public void setIndexType(IndexType indexType) { + this.indexType = indexType; + } + + public String buildIndexScript(TableIndex tableIndex) { + StringBuilder script = new StringBuilder(); + + script.append(keyword).append(" "); + script.append(buildIndexName(tableIndex)).append(" "); + script.append(buildIndexColumn(tableIndex)).append(" "); + script.append(buildIndexType(tableIndex)).append(" "); + return script.toString(); + } + + private String buildIndexType(TableIndex tableIndex) { + if (this.equals(PRIMARY)) { + return ""; + } else { + return "TYPE " + name ; + } + } + + private String buildIndexColumn(TableIndex tableIndex) { + StringBuilder script = new StringBuilder(); + script.append("("); + for (TableIndexColumn column : tableIndex.getColumnList()) { + if (StringUtils.isNotBlank(column.getColumnName())) { + script.append("`").append(column.getColumnName()).append("`"); + script.append(","); + } + } + script.deleteCharAt(script.length() - 1); + script.append(")"); + return script.toString(); + } + + private String buildIndexName(TableIndex tableIndex) { + if (this.equals(PRIMARY)) { + return ""; + } else { + return "`" + tableIndex.getName() + "`"; + } + } + + public String buildModifyIndex(TableIndex tableIndex) { + if (this.equals(PRIMARY)) { + return ""; + } + if (EditStatus.DELETE.name().equals(tableIndex.getEditStatus())) { + return StringUtils.join("DROP INDEX `", tableIndex.getOldName(), "`"); + } + if (EditStatus.MODIFY.name().equals(tableIndex.getEditStatus())) { + return StringUtils.join("DROP INDEX `", tableIndex.getOldName(), + "`,\n ADD ", buildIndexScript(tableIndex)); + } + if (EditStatus.ADD.name().equals(tableIndex.getEditStatus())) { + return StringUtils.join("ADD ", buildIndexScript(tableIndex)); + } + return ""; + } + +} diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java index 1454772d..a0d164cb 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java @@ -1,12 +1,5 @@ package ai.chat2db.server.domain.core.impl; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; - import ai.chat2db.server.domain.api.enums.TableVectorEnum; import ai.chat2db.server.domain.api.param.*; import ai.chat2db.server.domain.api.service.PinService; @@ -15,7 +8,9 @@ import ai.chat2db.server.domain.core.cache.CacheManage; import ai.chat2db.server.domain.core.converter.PinTableConverter; import ai.chat2db.server.domain.core.converter.TableConverter; import ai.chat2db.server.domain.repository.Dbutils; -import ai.chat2db.server.domain.repository.entity.*; +import ai.chat2db.server.domain.repository.entity.TableCacheDO; +import ai.chat2db.server.domain.repository.entity.TableCacheVersionDO; +import ai.chat2db.server.domain.repository.entity.TableVectorMappingDO; import ai.chat2db.server.domain.repository.mapper.TableCacheMapper; import ai.chat2db.server.domain.repository.mapper.TableCacheVersionMapper; import ai.chat2db.server.domain.repository.mapper.TableVectorMappingMapper; @@ -40,6 +35,13 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + import static ai.chat2db.server.domain.core.cache.CacheKey.getColumnKey; import static ai.chat2db.server.domain.core.cache.CacheKey.getTableKey; @@ -564,7 +566,7 @@ public class TableServiceImpl implements TableService { //filter primary key List indexTypes = tableMeta.getIndexTypes(); if (CollectionUtils.isNotEmpty(indexTypes)) { - List types = indexTypes.stream().filter(indexType -> !"Primary".equals(indexType.getTypeName())).collect(Collectors.toList()); + List types = indexTypes.stream().filter(indexType -> !"Primary".equalsIgnoreCase(indexType.getTypeName())).collect(Collectors.toList()); tableMeta.setIndexTypes(types); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/EngineType.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/EngineType.java new file mode 100644 index 00000000..c923cd96 --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/EngineType.java @@ -0,0 +1,19 @@ +package ai.chat2db.spi.model; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class EngineType { + private String name; + private boolean supportTTL; + private boolean supportSortOrder; + private boolean supportSkippingIndices; + private boolean supportDeduplication; + private boolean supportSettings; + private boolean supportParallelInsert; + private boolean supportProjections; + private boolean supportReplication; + +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Table.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Table.java index 640f2180..397dc9b4 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Table.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Table.java @@ -1,14 +1,13 @@ package ai.chat2db.spi.model; -import java.util.List; - - import com.fasterxml.jackson.annotation.JsonAlias; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; +import java.util.List; + /** * 表信息 * @@ -77,7 +76,10 @@ public class Table { */ private String ddl; - + /** + * engine + */ + @JsonAlias("TYPE_NAME") private String engine; diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/TableMeta.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/TableMeta.java index 6ae0c2d7..a524ad74 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/TableMeta.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/TableMeta.java @@ -21,4 +21,6 @@ public class TableMeta { private List indexTypes; private List defaultValues; + + private List engineTypes; }