Merge pull request #2802 from beekeeper-studio/feat/config-binary-encoding

Feat: Support binary encoding configuration
This commit is contained in:
Mohammad Azmi
2025-04-02 18:10:00 +07:00
committed by GitHub
22 changed files with 130 additions and 155 deletions

View File

@@ -3,6 +3,11 @@ checkForUpdatesInterval = 86400000 ; 24 hours
dataSyncInterval = 30000 ; 30 secs
workspaceSyncInterval = 5000 ; 5 seconds
[ui.general]
; Controls the encoding format used to display binary data in the application.
; Possible values: base64, hex
binaryEncoding = hex
[ui.queryEditor]
maxResults = 50000

View File

@@ -19,7 +19,7 @@
"lint": "eslint",
"electron:build": "yarn build && yarn electron-builder --config ./electron-builder-config.js",
"electron:serve": "concurrently -c blue,green -n esbuild,vite 'yarn dev:esbuild' 'yarn dev:vite'",
"config:build": "CLI_MODE=1 node ./typesGenerator.mjs",
"config:build": "CLI_MODE=1 node ./src/config/typesGenerator.mjs",
"postinstall": "electron-builder install-app-deps",
"internal:integration": "cross-env TEST_MODE=1 ELECTRON_RUN_AS_NODE=1 yarn electron ../../node_modules/jest/bin/jest.js --config ./jest.integration.config.js",
"test:integration": "../../bin/integration-tests.sh",

View File

@@ -1,3 +1,9 @@
// Uint8Array pollyfills https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array/fromBase64
import 'core-js/actual/typed-array/from-base64'
import 'core-js/actual/typed-array/from-hex'
import 'core-js/actual/typed-array/to-base64'
import 'core-js/actual/typed-array/to-hex'
import Vue from 'vue'
import VueHotkey from 'v-hotkey'
import VModal from 'vue-js-modal'

View File

@@ -54,7 +54,10 @@ export function tabulatorForTableData(
width: 38,
hozAlign: "center",
formatter: "rownum",
formatterParams: { relativeToPage: true },
formatterParams: {
relativeToPage: true,
binaryEncoding: window.bksConfig.ui.general.binaryEncoding,
},
contextMenu: (_e, cell) => {
return copyActionsMenu({ ranges: cell.getRanges(), table: table || "mytable", schema });
},

View File

@@ -182,9 +182,15 @@ export function stringifyRangeData(rangeData: Record<string, any>[]) {
transformedRangeData[i] = {};
for (const key of keys) {
const value = rangeData[i][key];
transformedRangeData[i][key] =
value && typeof value === "object" ? JSON.stringify(value) : value;
let value = rangeData[i][key];
if (_.isTypedArray(value)) {
value = typedArrayToString(value);
} else if (value && typeof value === "object") {
value = JSON.stringify(value);
}
transformedRangeData[i][key] = value
}
}
@@ -217,82 +223,6 @@ export function streamToBuffer(stream: Stream): Promise<Buffer> {
});
}
/**
* Extracted from https://github.com/zloirock/core-js/blob/master/packages/core-js/modules/esnext.uint8-array.to-hex.js
* FIXME we don't need this soon after `UInt8Array.prototype.toHex` is
* implemented. See https://github.com/tc39/proposal-arraybuffer-base64
*/
export function uint8ArrayToHex(arr: Uint8Array): string {
let result = '';
for (var i = 0, length = arr.length; i < length; i++) {
var hex = 1.0.toString.call(arr[i], 16);
result += hex.length === 1 ? '0' + hex : hex;
}
return result;
}
function uncurryThis(fn: any): any {
return function () {
return Function.prototype.call.apply(fn, arguments);
};
}
var min = Math.min;
var NOT_HEX = /[^\da-f]/i;
var exec = uncurryThis(NOT_HEX.exec);
var stringSlice = uncurryThis(''.slice);
/**
* Extracted from https://github.com/zloirock/core-js/blob/master/packages/core-js/internals/uint8-from-hex.js
* FIXME we don't need this soon after `UInt8Array.prototype.fromHex` is
* implemented. See https://github.com/tc39/proposal-arraybuffer-base64
*/
export function hexToUint8Array(string: string, into?: any): Uint8Array {
var stringLength = string.length;
if (stringLength % 2 !== 0) throw new SyntaxError('String should be an even number of characters');
var maxLength = into ? min(into.length, stringLength / 2) : stringLength / 2;
var bytes = into || new Uint8Array(maxLength);
var read = 0;
var written = 0;
while (written < maxLength) {
var hexits = stringSlice(string, read, read += 2);
if (exec(NOT_HEX, hexits)) throw new SyntaxError('String should only contain hex characters');
bytes[written++] = parseInt(hexits, 16);
}
return bytes
}
type FriendlyUint8Array = Uint8Array & {
toString(): string
toJSON(): string
toHex(): string
toBase64(): string
}
/** Make `Uint8Array.toString` look better :D */
export function friendlyUint8Array(bytes: string, encoding: 'hex'): FriendlyUint8Array
export function friendlyUint8Array(bytes: Uint8Array): FriendlyUint8Array
export function friendlyUint8Array(bytes: string | Uint8Array, encoding?: 'hex'): FriendlyUint8Array {
if (typeof bytes === 'string') {
bytes = hexToUint8Array(bytes)
}
Object.assign(bytes, {
toString() {
return uint8ArrayToHex(this)
},
toJSON() {
return uint8ArrayToHex(this)
},
toHex() {
return uint8ArrayToHex(this)
},
toBase64() {
return '[TODO]'
}
})
return bytes as FriendlyUint8Array
}
/** Make `object.toString` look better :D */
export function friendlyJsonObject<T extends object>(obj: T): T {
Object.defineProperties(obj, {
@@ -321,7 +251,7 @@ export function friendlyJsonObject<T extends object>(obj: T): T {
}
})
}
return obj;
}
@@ -332,3 +262,25 @@ export function stringifyWithBigInt(value: any): string {
);
}
/** Convert Typed Array (Array Buffer View) to string based on `binaryEncoding` */
export function typedArrayToString(typedArray: ArrayBufferView, forceEncoding?: 'hex' | 'base64') {
const encoding = forceEncoding || window.bksConfig.ui.general.binaryEncoding
if (encoding === 'base64') {
// @ts-expect-error polyfill
return typedArray.toBase64();
} else {
// @ts-expect-error polyfill
return typedArray.toHex();
}
}
export function stringToTypedArray(str: string, forceEncoding?: 'hex' | 'base64') {
const encoding = forceEncoding || window.bksConfig.ui.general.binaryEncoding
if (encoding === 'base64') {
// @ts-expect-error polyfill
return Uint8Array.fromBase64(str);
} else {
// @ts-expect-error polyfill
return Uint8Array.fromHex(str);
}
}

View File

@@ -149,6 +149,7 @@
:result="result"
:query="query"
:tab="tab"
:binaryEncoding="$bksConfig.ui.general.binaryEncoding"
/>
<div
class="message"

View File

@@ -37,7 +37,7 @@
actualTableHeight: '100%',
}
},
props: ['result', 'tableHeight', 'query', 'active', 'tab', 'focus'],
props: ['result', 'tableHeight', 'query', 'active', 'tab', 'focus', 'binaryEncoding'],
watch: {
active() {
if (!this.tabulator) return;
@@ -119,6 +119,9 @@
const magicStuff = _.pick(magic, ['formatter', 'formatterParams'])
const defaults = {
formatter: this.cellFormatter,
formatterParams: {
binaryEncoding: this.binaryEncoding,
},
}
const result = {

View File

@@ -98,12 +98,13 @@ import DetailViewSidebarUpsell from '@/components/upsell/DetailViewSidebarUpsell
import rawLog from "@bksLogger";
import _ from "lodash";
import globals from '@/common/globals'
import { typedArrayToString } from '@/common/utils'
const log = rawLog.scope("detail-view-sidebar");
export default Vue.extend({
components: { TextEditor, DetailViewSidebarUpsell },
props: ["value", "hidden", "expandablePaths", "dataId", "title", "reinitialize", "signs"],
props: ["value", "hidden", "expandablePaths", "dataId", "title", "reinitialize", "signs", "binaryEncoding"],
data() {
return {
reinitializeTextEditor: 0,
@@ -147,9 +148,9 @@ export default Vue.extend({
}
if (this.filter) {
const filtered = deepFilterObjectProps(this.processedValue, this.filter);
return JSON.stringify(filtered, null, 2);
return JSON.stringify(filtered, this.replacer, 2);
}
return JSON.stringify(this.processedValue, null, 2);
return JSON.stringify(this.processedValue, this.replacer, 2);
},
debouncedFilter: {
get() {
@@ -285,6 +286,12 @@ export default Vue.extend({
...mapGetters(["expandFKDetailsByDefault"]),
},
methods: {
replacer(_key: string, value: unknown) {
if (_.isTypedArray(value)) {
return typedArrayToString(value as ArrayBufferView, this.binaryEncoding)
}
return value
},
expandPath(path: ExpandablePath) {
this.$emit("expandPath", path);
},

View File

@@ -155,9 +155,13 @@ import { uuidv4 } from "@/lib/uuid"
import _ from 'lodash'
import { mapGetters } from 'vuex'
import TextEditor from '@/components/common/texteditor/TextEditor.vue'
import { typedArrayToString } from "@/common/utils";
export default Vue.extend({
name: "CellEditorModal",
props: {
binaryEncoding: String,
},
data() {
return {
editorFocus: false,
@@ -202,8 +206,8 @@ export default Vue.extend({
openModal(content: any, language: LanguageData, eventParams?: any) {
if (content === null) {
content = ""
} else if (ArrayBuffer.isView(content)) {
content = content.toString()
} else if (_.isTypedArray(content)) {
content = typedArrayToString(content, this.binaryEncoding)
} else if (typeof content !== 'string') {
content = JSON.stringify(content)
}

View File

@@ -6,6 +6,7 @@
>
<editor-modal
ref="editorModal"
:binary-encoding="$bksConfig.ui.general.binaryEncoding"
@save="onSaveEditorModal"
/>
<template v-if="!table && initialized">
@@ -41,6 +42,7 @@
:expandable-paths="expandablePaths"
:reinitialize="reinitializeDetailView"
:signs="selectedRowDataSigns"
:binary-encoding="$bksConfig.ui.general.binaryEncoding"
@expandPath="expandForeignKey"
@close="toggleOpenDetailView(false)"
/>
@@ -340,7 +342,7 @@ import { getFilters, setFilters } from "@/common/transport/TransportOpenTab"
import DetailViewSidebar from '@/components/sidebar/DetailViewSidebar.vue'
import Split from 'split.js'
import { ExpandablePath } from '@/lib/data/detail_view'
import { hexToUint8Array, friendlyUint8Array } from '@/common/utils';
import { stringToTypedArray } from "@/common/utils";
const log = rawLog.scope('TableTable')
@@ -839,7 +841,8 @@ export default Vue.extend({
formatterParams: {
fk: hasKeyDatas && keyDatas[0][1],
fkOnClick: hasKeyDatas && ((_e, cell) => this.fkClick(keyDatas[0][1][0], cell)),
isPK: isPK
isPK: isPK,
binaryEncoding: this.$bksConfig.ui.general.binaryEncoding,
},
editorParams: {
verticalNavigation: useVerticalNavigation ? 'editor' : undefined,
@@ -853,6 +856,7 @@ export default Vue.extend({
},
typeHint: column.dataType?.toLowerCase(),
bksField: column.bksField,
binaryEncoding: this.$bksConfig.ui.general.binaryEncoding,
},
}
@@ -1088,35 +1092,38 @@ export default Vue.extend({
})
}
},
openCellEditorModal(cell: CellComponent, isReadOnly: boolean) {
const eventParams = { cell, isReadOnly };
this.$refs.editorModal.openModal(cell.getValue(), undefined, eventParams)
},
openEditorMenuByShortcut() {
const range: RangeComponent = _.last(this.tabulator.getRanges())
const cell = range.getCells().flat()[0];
// FIXME maybe we can avoid calling child methods directly like this?
// it should be done by calling an event using this.$modal.show(modalName)
// or this.$trigger(AppEvent.something) if possible
if (this.isPrimaryKey(cell.getField())) return;
const eventParams = {
cell,
isReadOnly: this.isEditorMenuDisabled(cell)
};
this.$refs.editorModal.openModal(cell.getValue(), undefined, eventParams)
this.openCellEditorModal(cell, this.isEditorMenuDisabled(cell))
},
openEditorMenu(cell: CellComponent) {
const isReadOnly = this.isEditorMenuDisabled(cell);
return {
label: createMenuItem(isReadOnly? "View in modal" : "Edit in modal", "Shift + Enter"),
action: () => {
if (this.isPrimaryKey(cell.getField())) return
this.$refs.editorModal.openModal(cell.getValue(), undefined, { cell, isReadOnly })
this.openCellEditorModal(cell, isReadOnly)
}
}
},
onSaveEditorModal(content: string, _: LanguageData, cell: CellComponent){
if (ArrayBuffer.isView(cell.getValue())) {
cell.setValue(friendlyUint8Array(hexToUint8Array(content)))
} else {
cell.setValue(content)
onSaveEditorModal(content: string, _l: LanguageData, cell: CellComponent){
const column = this.table.columns.find((col) => col.columnName === cell.getField());
const isBinary = column?.bksField?.bksType === 'BINARY' || _.isTypedArray(cell.getValue());
let value = content;
if (isBinary) {
value = stringToTypedArray(content)
}
cell.setValue(value)
},
openProperties() {
this.$root.$emit(AppEvent.openTableProperties, { table: this.table })

View File

@@ -6,7 +6,7 @@ import fs from "fs";
import dtsGen from "dts-gen";
import path from "path";
import _ from "lodash";
import { parseIni } from "./helpers";
import { parseIni } from "./helpers.js";
function resolveRootPath() {
const __dirname = path.resolve();
@@ -14,7 +14,7 @@ function resolveRootPath() {
if (dirpath.includes("node_modules")) {
return dirpath.split("node_modules")[0];
}
return path.resolve(dirpath, "../..");
return dirpath;
}
export function generateConfigTypes() {
@@ -34,7 +34,7 @@ export function generateConfigTypes() {
.replace("declare const IBksConfig:", "declare interface IBksConfig");
fs.writeFileSync(
path.join(rootPath, "apps/studio/src/typings/bksConfig.d.ts"),
path.join(rootPath, "src/typings/bksConfig.d.ts"),
result
);
}

View File

@@ -1,6 +1,6 @@
import _ from 'lodash'
import { Dialect } from '@shared/lib/dialects/models'
import { friendlyUint8Array, uint8ArrayToHex, friendlyJsonObject } from '@/common/utils';
import { friendlyJsonObject } from '@/common/utils';
type JsonFriendly = string | boolean | number | null | JsonFriendly[] | Record<string, any>
function dec28bits(num: any): string {
@@ -50,7 +50,7 @@ export const Mutators = {
genericMutator(value: any, preserveComplex = false): JsonFriendly {
const mutate = Mutators.genericMutator
if (ArrayBuffer.isView(value)) {
return friendlyUint8Array(value as Uint8Array)
return value
}
if (_.isDate(value)) return value.toISOString()
if (_.isArray(value)) return preserveComplex? value.map((v) => mutate(v, preserveComplex)) : JSON.stringify(value)

View File

@@ -47,14 +47,15 @@ export default {
cellTooltip(_event, cell: CellComponent) {
let cellValue = cell.getValue()
if (cellValue instanceof Uint8Array) {
cellValue = `${_.truncate(cellValue.toString(), { length: 15 })} (as hex string)`
const binaryEncoding = cell.getColumn().getDefinition().formatterParams?.binaryEncoding || 'hex'
cellValue = `${_.truncate(this.niceString(cellValue, false, binaryEncoding), { length: 15 })} (as ${binaryEncoding} string)`
}
const nullValue = emptyResult(cellValue)
return nullValue ? nullValue : escapeHtml(this.niceString(cellValue, true))
},
cellFormatter(
cell: CellComponent,
params: { fk?: any[], isPK?: boolean, fkOnClick?: (e: MouseEvent, cell: CellComponent) => void },
params: { fk?: any[], isPK?: boolean, fkOnClick?: (e: MouseEvent, cell: CellComponent) => void, binaryEncoding?: string } = {},
onRendered: (func: () => void) => void
) {
const classNames = []
@@ -69,7 +70,7 @@ export default {
if (nullValue) {
return nullValue
}
cellValue = this.niceString(cellValue, true)
cellValue = this.niceString(cellValue, true, params.binaryEncoding)
cellValue = cellValue.replace(/\n/g, ' ↩ ');
// removing the <pre> will break selection / copy paste, see ResultTable

View File

@@ -24,8 +24,8 @@ import _ from 'lodash'
import Vue from 'vue'
import helpers from '@shared/lib/tabulator'
import rawLog from '@bksLogger'
import { hexToUint8Array, friendlyUint8Array } from '@/common/utils';
import { BksField } from "@/lib/db/models";
import { typedArrayToString } from "@/common/utils";
const log = rawLog.scope('NullableInputEditor')
@@ -118,8 +118,8 @@ export default Vue.extend({
parseValue() {
const typeHint = this.params.typeHint;
const bksField: BksField = this.params.bksField;
if (bksField?.bksType === 'BINARY' || ArrayBuffer.isView(this.cell.getValue())) {
return friendlyUint8Array(hexToUint8Array(this.value));
if (bksField?.bksType === 'BINARY' || _.isTypedArray(this.cell.getValue())) {
return typedArrayToString(this.value, this.params.binaryEncoding)
}
if (typeof typeHint !== 'string') {
return this.value

View File

@@ -1,3 +1,4 @@
import { typedArrayToString } from '@/common/utils';
import _ from 'lodash'
import {CellComponent} from 'tabulator-tables'
@@ -27,9 +28,11 @@ function yesNoResult(value: boolean) {
}
export default {
niceString(value: any, truncate = false) {
niceString(value: any, truncate = false, binaryEncoding?: 'hex' | 'base64') {
let cellValue = value.toString();
if(_.isArray(value) || (_.isObject(value) && !ArrayBuffer.isView(value))) {
if (_.isTypedArray(value)) {
cellValue = typedArrayToString(value, binaryEncoding)
} else if (_.isArray(value) || _.isObject(value)) {
cellValue = JSON.stringify(value)
}
return truncate ? _.truncate(cellValue, { length: 256 }) : cellValue

View File

@@ -77,6 +77,7 @@ declare interface IBksConfig {
tableTable: {
focusOnFilterInput: string;
nextPage: string;
openEditorModal: string;
previousPage: string;
};
};
@@ -84,6 +85,9 @@ declare interface IBksConfig {
export: {
errorNoticeTimeout: number;
};
general: {
binaryEncoding: string;
};
queryEditor: {
maxResults: number;
};

View File

@@ -7,7 +7,7 @@ describe("cellFormatter", () => {
const input = {
getValue: () => '<a>foo</a>',
getElement: () => document.createElement('a'),
getColumn: () => ({ getDefinition: () => ({ binaryEncoding: 'base64' }) }),
}
const formatted = mutators.methods.cellFormatter(input)
@@ -20,11 +20,13 @@ describe("cellFormatter", () => {
const input = {
getValue: () => '8640000000000000',
getElement: () => document.createElement('a'),
getColumn: () => ({ getDefinition: () => ({ binaryEncoding: 'base64' }) }),
}
const badInput = {
getValue: () => '8640000000000005',
getElement: () => document.createElement('a'),
getColumn: () => ({ getDefinition: () => ({ binaryEncoding: 'base64' }) }),
}
const params = {

View File

@@ -80,6 +80,10 @@ export default Vue.extend({
type: Array as PropType<Array<OrderBy>>,
default: () => [],
},
binaryEncoding: {
type: String as PropType<'hex' | 'base64'>,
default: 'hex',
},
cellContextMenuItems: [Array, Function] as PropType<CustomMenuItems>,
columnHeaderContextMenuItems: [Array, Function] as PropType<CustomMenuItems>,
rowHeaderContextMenuItems: [Array, Function] as PropType<CustomMenuItems>,
@@ -203,6 +207,7 @@ export default Vue.extend({
resizable: "header",
cssClass,
sorter: column.sorter === 'none' ? () => 0 : column.sorter,
binaryEncoding: this.binaryEncoding,
};
const customDef =

View File

@@ -45,16 +45,17 @@ export default {
return cellValue.map(cv => `<span class="mapper-pill">${cv}</span>`).join('')
},
cellTooltip(_event, cell: CellComponent) {
const binaryEncoding = cell.getColumn().getDefinition().formatterParams.binaryEncoding
let cellValue = cell.getValue()
if (cellValue instanceof Uint8Array) {
cellValue = `${_.truncate(cellValue.toString(), { length: 15 })} (as hex string)`
cellValue = `${_.truncate(this.niceString(cellValue, false, binaryEncoding), { length: 15 })} (as ${binaryEncoding} string)`
}
const nullValue = emptyResult(cellValue)
return nullValue ? nullValue : escapeHtml(this.niceString(cellValue, true))
},
cellFormatter(
cell: CellComponent,
params: { fk?: any[], isPK?: boolean, fkOnClick?: (value: any, field: string, cell: CellComponent) => void },
params: { fk?: any[], isPK?: boolean, fkOnClick?: (value: any, field: string, cell: CellComponent) => void, binaryEncoding?: 'hex' | 'base64' } = {},
onRendered: (func: () => void) => void
) {
const classNames = []
@@ -69,7 +70,7 @@ export default {
if (nullValue) {
return nullValue
}
cellValue = this.niceString(cellValue, true)
cellValue = this.niceString(cellValue, true, params.binaryEncoding)
cellValue = cellValue.replace(/\n/g, ' ↩ ');
// removing the <pre> will break selection / copy paste, see ResultTable

View File

@@ -2,7 +2,6 @@ import _ from 'lodash'
// import { Dialect } from '@shared/lib/dialects/models'
// FIXME cant import Dialect type from here
type Dialect = any
import { friendlyUint8Array, uint8ArrayToHex } from '../../../utils/binary';
type JsonFriendly = string | boolean | number | null | JsonFriendly[] | Record<string, any>
function dec28bits(num: any): string {
@@ -52,7 +51,7 @@ export const Mutators = {
genericMutator(value: any, preserveComplex = false): JsonFriendly {
const mutate = Mutators.genericMutator
if (ArrayBuffer.isView(value)) {
return friendlyUint8Array(value as Uint8Array)
return value
}
if (_.isDate(value)) return value.toISOString()
if (_.isArray(value)) return preserveComplex? value.map((v) => mutate(v, preserveComplex)) : JSON.stringify(value)

View File

@@ -27,9 +27,12 @@ function yesNoResult(value: boolean) {
}
export default {
niceString(value: any, truncate = false) {
niceString(value: any, truncate = false, binaryEncoding?: 'hex' | 'base64') {
let cellValue = value.toString();
if(_.isArray(value) || (_.isObject(value) && !ArrayBuffer.isView(value))) {
if (value instanceof Uint8Array) {
// @ts-expect-error polyfilled
cellValue = binaryEncoding === 'base64' ? value.toBase64() : value.toHex()
} else if (_.isArray(value) || (_.isObject(value) && !ArrayBuffer.isView(value))) {
cellValue = JSON.stringify(value)
}
return truncate ? _.truncate(cellValue, { length: 256 }) : cellValue

View File

@@ -42,34 +42,3 @@ export function hexToUint8Array(string: string, into?: any): Uint8Array {
}
return bytes
}
type FriendlyUint8Array = Uint8Array & {
toString(): string
toJSON(): string
toHex(): string
toBase64(): string
}
/** Make `Uint8Array.toString` look better :D */
export function friendlyUint8Array(bytes: string, encoding: 'hex'): FriendlyUint8Array
export function friendlyUint8Array(bytes: Uint8Array): FriendlyUint8Array
export function friendlyUint8Array(bytes: string | Uint8Array, encoding?: 'hex'): FriendlyUint8Array {
if (typeof bytes === 'string') {
bytes = hexToUint8Array(bytes)
}
Object.assign(bytes, {
toString() {
return uint8ArrayToHex(this)
},
toJSON() {
return uint8ArrayToHex(this)
},
toHex() {
return uint8ArrayToHex(this)
},
toBase64() {
return '[TODO]'
}
})
return bytes as FriendlyUint8Array
}