From 854efdf7340aef8d60f4ff824b30bbbbce247ea0 Mon Sep 17 00:00:00 2001 From: Ansgar Becker Date: Mon, 9 Sep 2024 17:40:53 +0200 Subject: [PATCH] Issue #2009: basic implementation of a data generation tool, in table tools dialog --- out/locale/en/LC_MESSAGES/default.po | 10 +- source/apphelpers.pas | 3 + source/main.dfm | 12 ++ source/main.pas | 7 +- source/tabletools.dfm | 54 ++++++++ source/tabletools.pas | 192 ++++++++++++++++++++++++++- 6 files changed, 272 insertions(+), 6 deletions(-) diff --git a/out/locale/en/LC_MESSAGES/default.po b/out/locale/en/LC_MESSAGES/default.po index 214c7279..b771b415 100644 --- a/out/locale/en/LC_MESSAGES/default.po +++ b/out/locale/en/LC_MESSAGES/default.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: HeidiSQL\n" "POT-Creation-Date: 2012-11-05 21:40\n" -"PO-Revision-Date: 2024-08-16 12:20+0200\n" +"PO-Revision-Date: 2024-09-09 17:37+0200\n" "Last-Translator: Ansgar Becker \n" "Language-Team: English (http://www.transifex.com/projects/p/heidisql/language/en/)\n" "Language: en\n" @@ -15,7 +15,7 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" -"X-Generator: Poedit 3.4.4\n" +"X-Generator: Poedit 3.5\n" #. AboutBox..Caption #: about.dfm:5 @@ -6705,3 +6705,9 @@ msgstr "Warning: Failed to set cipher encryption parameter \"%s\"" msgid "You have activated encryption on a probably non-encrypted database." msgstr "You have activated encryption on a probably non-encrypted database." + +msgid "Number of rows:" +msgstr "Number of rows:" + +msgid "Amount of NULLs [percent]:" +msgstr "Amount of NULLs [percent]:" diff --git a/source/apphelpers.pas b/source/apphelpers.pas index 34ce7388..d99d6193 100644 --- a/source/apphelpers.pas +++ b/source/apphelpers.pas @@ -233,6 +233,7 @@ type asCreateDbCollation, asRealTrailingZeros, asSequalSuggestWindowWidth, asSequalSuggestWindowHeight, asSequalSuggestPrompt, asSequalSuggestRecentPrompts, asReformatter, asAlwaysGenerateFilter, + asGenerateDataNumRows, asGenerateDataNullAmount, asUnused); TAppSetting = record Name: String; @@ -3846,6 +3847,8 @@ begin InitSetting(asSequalSuggestRecentPrompts, 'SequalSuggestRecentPrompts', 0, False, ''); InitSetting(asReformatter, 'Reformatter', 0); InitSetting(asAlwaysGenerateFilter, 'AlwaysGenerateFilter', 0, False); + InitSetting(asGenerateDataNumRows, 'GenerateDataNumRows', 1000); + InitSetting(asGenerateDataNullAmount, 'GenerateDataNullAmount', 10); // Default folder for snippets if FPortableMode then diff --git a/source/main.dfm b/source/main.dfm index 1e1e0f3a..b177d666 100644 --- a/source/main.dfm +++ b/source/main.dfm @@ -2021,6 +2021,9 @@ object MainForm: TMainForm object Bulktableeditor1: TMenuItem Action = actBulkTableEdit end + object Generatedata1: TMenuItem + Action = actGenerateData + end object Launchcommandline1: TMenuItem Action = actLaunchCommandline end @@ -2989,6 +2992,12 @@ object MainForm: TMainForm ImageName = 'icons8-data-backup' OnExecute = actSynchronizeDatabaseExecute end + object actGenerateData: TAction + Category = 'Tools' + Caption = 'Generate data' + ImageIndex = 130 + OnExecute = actTableToolsExecute + end object actLaunchCommandline: TAction Category = 'Tools' Caption = 'Launch command line' @@ -3453,6 +3462,9 @@ object MainForm: TMainForm object menuBulkTableEdit: TMenuItem Action = actBulkTableEdit end + object Generatedata2: TMenuItem + Action = actGenerateData + end object N5a: TMenuItem Caption = '-' end diff --git a/source/main.pas b/source/main.pas index 3b581712..7434b4e3 100644 --- a/source/main.pas +++ b/source/main.pas @@ -790,6 +790,9 @@ type Resetpaneldimensions1: TMenuItem; popupApplyFilter: TPopupMenu; menuAlwaysGenerateFilter: TMenuItem; + actGenerateData: TAction; + Generatedata1: TMenuItem; + Generatedata2: TMenuItem; procedure actCreateDBObjectExecute(Sender: TObject); procedure menuConnectionsPopup(Sender: TObject); procedure actExitApplicationExecute(Sender: TObject); @@ -2947,7 +2950,9 @@ begin else if Sender = actExportTables then FTableToolsDialog.ToolMode := tmSQLExport else if Sender = actBulkTableEdit then - FTableToolsDialog.ToolMode := tmBulkTableEdit; + FTableToolsDialog.ToolMode := tmBulkTableEdit + else if Sender = actGenerateData then + FTableToolsDialog.ToolMode := tmGenerateData; FTableToolsDialog.ShowModal; FreeAndNil(FTableToolsDialog); end; diff --git a/source/tabletools.dfm b/source/tabletools.dfm index d348bcd5..d4bb3eeb 100644 --- a/source/tabletools.dfm +++ b/source/tabletools.dfm @@ -605,6 +605,60 @@ object frmTableTools: TfrmTableTools TabOrder = 8 end end + object tabGenerateData: TTabSheet + Caption = 'Generate data' + ImageIndex = 130 + object lblGenerateDataNumRows: TLabel + Left = 3 + Top = 6 + Width = 92 + Height = 14 + Caption = 'Number of rows:' + end + object lblGenerateDataNullAmount: TLabel + Left = 2 + Top = 34 + Width = 126 + Height = 14 + Caption = 'Amount of NULLs [percent]:' + end + object editGenerateDataNumRows: TEdit + Left = 200 + Top = 3 + Width = 121 + Height = 22 + TabOrder = 0 + Text = '1.000' + end + object updownGenerateDataNumRows: TUpDown + Left = 321 + Top = 3 + Width = 20 + Height = 22 + Associate = editGenerateDataNumRows + Min = 1 + Max = 2147483647 + Position = 1000 + TabOrder = 1 + end + object editGenerateDataNullAmount: TEdit + Left = 200 + Top = 31 + Width = 121 + Height = 22 + TabOrder = 2 + Text = '10' + end + object updownGenerateDataNullAmount: TUpDown + Left = 321 + Top = 31 + Width = 20 + Height = 22 + Associate = editGenerateDataNullAmount + Position = 10 + TabOrder = 3 + end + end end end object pnlLeft: TPanel diff --git a/source/tabletools.pas b/source/tabletools.pas index 434ffc8e..36ee3225 100644 --- a/source/tabletools.pas +++ b/source/tabletools.pas @@ -13,10 +13,10 @@ uses VirtualTrees, Vcl.ExtCtrls, Vcl.Graphics, SynRegExpr, System.Math, System.Generics.Collections, extra_controls, dbconnection, apphelpers, Vcl.Menus, gnugettext, System.DateUtils, System.Zip, System.UITypes, System.StrUtils, Winapi.Messages, SynEdit, SynMemo, Vcl.ClipBrd, generic_types, VirtualTrees.Types, VirtualTrees.BaseAncestorVCL, - VirtualTrees.BaseTree, VirtualTrees.AncestorVCL; + VirtualTrees.BaseTree, VirtualTrees.AncestorVCL, System.JSON, System.Variants; type - TToolMode = (tmMaintenance, tmFind, tmSQLExport, tmBulkTableEdit); + TToolMode = (tmMaintenance, tmFind, tmSQLExport, tmBulkTableEdit, tmGenerateData); TfrmTableTools = class(TExtForm) btnCloseOrCancel: TButton; pnlTop: TPanel; @@ -95,6 +95,13 @@ type editTableFilter: TButtonedEdit; TreeObjects: TVirtualStringTree; timerCalcSize: TTimer; + tabGenerateData: TTabSheet; + lblGenerateDataNumRows: TLabel; + editGenerateDataNumRows: TEdit; + updownGenerateDataNumRows: TUpDown; + lblGenerateDataNullAmount: TLabel; + editGenerateDataNullAmount: TEdit; + updownGenerateDataNullAmount: TUpDown; procedure FormCreate(Sender: TObject); procedure FormShow(Sender: TObject); procedure btnHelpMaintenanceClick(Sender: TObject); @@ -180,6 +187,7 @@ type procedure DoFind(DBObj: TDBObject); procedure DoExport(DBObj: TDBObject); procedure DoBulkTableEdit(DBObj: TDBObject); + procedure DoGenerateData(DBObj: TDBObject); public { Public declarations } PreSelectObjects: TDBObjectList; @@ -297,6 +305,10 @@ begin SessionPaths.Free; comboExportOutputTarget.Text := ''; + // Generate data tab + updownGenerateDataNumRows.Position := AppSettings.ReadInt(asGenerateDataNumRows); + updownGenerateDataNullAmount.Position := AppSettings.ReadInt(asGenerateDataNullAmount); + // Various FixVT(TreeObjects); FixVT(ResultGrid); @@ -544,6 +556,11 @@ begin end; end; + tmGenerateData: begin + AppSettings.WriteInt(asGenerateDataNumRows, updownGenerateDataNumRows.Position); + AppSettings.WriteInt(asGenerateDataNullAmount, updownGenerateDataNullAmount.Position); + end; + end; end; @@ -606,7 +623,11 @@ begin OptionChecked := chkBulkTableEditDatabase.Checked or chkBulkTableEditEngine.Checked or chkBulkTableEditCollation.Checked or chkBulkTableEditCharset.Checked or chkBulkTableEditResetAutoinc.Checked; btnExecute.Enabled := SomeChecked and OptionChecked; + end else if tabsTools.ActivePage = tabGenerateData then begin + btnExecute.Caption := _('Generate'); + btnExecute.Enabled := SomeChecked; end; + end; @@ -787,6 +808,7 @@ var tmFind: DoFind(DBObj); tmSQLExport: DoExport(DBObj); tmBulkTableEdit: DoBulkTableEdit(DBObj); + tmGenerateData: DoGenerateData(DBObj); end; except on E:EDbError do begin @@ -827,7 +849,9 @@ begin else if tabsTools.ActivePage = tabSQLExport then FToolMode := tmSQLExport else if tabsTools.ActivePage = tabBulkTableEdit then - FToolMode := tmBulkTableEdit; + FToolMode := tmBulkTableEdit + else if tabsTools.ActivePage = tabGenerateData then + FToolMode := tmGenerateData; ResultGrid.Clear; ResultGrid.TrySetFocus; FResults.Clear; @@ -1634,6 +1658,7 @@ begin tmFind: tabsTools.ActivePage := tabFind; tmSQLExport: tabsTools.ActivePage := tabSQLExport; tmBulkTableEdit: tabsTools.ActivePage := tabBulkTableEdit; + tmGenerateData: tabsTools.ActivePage := tabGenerateData; end; end; @@ -2152,6 +2177,167 @@ begin end; +procedure TfrmTableTools.DoGenerateData(DBObj: TDBObject); +var + Columns: TTableColumnList; + Col: TTableColumn; + InsertSqlBase, InsertSql: String; + ColumnNamesSkipped, ColumnNamesQuoted, Values: TStringList; + i, j: Integer; + IntVal, MaxLen, MinLen: Integer; + FloatVal: Extended; + JsonText: TJSONString; + EnumValues: TStringList; + TextVal: String; +begin + // Generate rows + if not (DBObj.NodeType in [lntTable, lntView]) then begin + AddNotes(DBObj, STRSKIPPED+'cannot insert rows in a '+LowerCase(DBObj.ObjType), ''); + Exit; + end; + AddNotes(DBObj, 'Inserting '+FormatNumber(updownGenerateDataNumRows.Position)+' rows into '+DBObj.Name, ''); + UpdateResultGrid; + + Columns := DBObj.TableColumns; + + InsertSqlBase := 'INSERT INTO ' + DBObj.QuotedDbAndTableName + ' '; + ColumnNamesQuoted := TStringList.Create; + ColumnNamesSkipped := TStringList.Create; + Values := TStringList.Create; + for Col in Columns do begin + if Col.DefaultType = cdtAutoInc then begin + ColumnNamesSkipped.Add(Col.Name); + Continue; + end; + if (Col.DefaultType = cdtExpression) and ExecRegExprI('^(NOW()|CURRENT_TIMESTAMP)', Col.DefaultText) then begin + ColumnNamesSkipped.Add(Col.Name); + Continue; + end; + + ColumnNamesQuoted.Add(Col.Connection.QuoteIdent(Col.Name)); + end; + InsertSqlBase := InsertSqlBase + '(' + Implode(', ', ColumnNamesQuoted) + ') VALUES '; + + Randomize; + + for i:=1 to updownGenerateDataNumRows.Position do begin + Values.Clear; + // Generate random values. Include some NULLs for columns which allow that. + for Col in Columns do begin + if ColumnNamesSkipped.Contains(Col.Name) then + Continue; + + // https://www.delphipraxis.net/31059-warscheinlichkeit-random.html + if Col.AllowNull + and (updownGenerateDataNullAmount.Position > 0) // prevent division by zero + and (Random < (updownGenerateDataNullAmount.Position / 100)) + then begin + Values.Add('NULL'); + Continue; + end; + + case Col.DataType.Category of + dtcInteger: begin + // Take care of overflow in RandomRange with signed integers + IntVal := 0; + case Col.DataType.Index of + dbdtTinyint: + IntVal := IfThen(Col.Unsigned, RandomRange(0, 256), RandomRange(-128, 128)); + dbdtSmallint: + IntVal := IfThen(Col.Unsigned, RandomRange(0, 65535), RandomRange(-32768, 32768)); + dbdtMediumint: + IntVal := IfThen(Col.Unsigned, RandomRange(0, 16777215), RandomRange(-8388608, 8388608)); + dbdtUint: + IntVal := RandomRange(0, MaxInt); + dbdtInt, dbdtBigint: + IntVal := IfThen(Col.Unsigned, RandomRange(0, MaxInt), RandomRange(0 - MaxInt, MaxInt)); + end; + Values.Add(IntVal.ToString); + end; + + dtcReal: begin + FloatVal := 0; + case Col.DataType.Index of + dbdtFloat, dbdtDouble, dbdtDecimal, dbdtNumeric, dbdtReal, dbdtDoublePrecision, dbdtMoney, dbdtSmallmoney: + FloatVal := IfThen(Col.Unsigned, RandomRange(0, 100000), RandomRange(-100000, 100000)) + Random; + end; + Values.Add(FloatToStr(FloatVal, MainForm.FormatSettings)); + end; + + dtcText: begin + MaxLen := 0; + case Col.DataType.Index of + dbdtChar, dbdtVarchar: + MaxLen := StrToIntDef(Col.LengthSet, 1); + dbdtTinytext: + MaxLen := Trunc(Power(2, 8)) -1; + dbdtText, dbdtMediumtext, dbdtLongtext, dbdtJson, dbdtJsonB: + MaxLen := Trunc(Power(2, 16)) -1; + end; + TextVal := ''; + MaxLen := RandomRange(1, MaxLen+1); + for j:=1 to MaxLen do begin + // Only printable characters + TextVal := TextVal + Chr(RandomRange(32, 127)); + end; + if Col.DataType.Index in [dbdtJson, dbdtJsonB] then begin + JsonText := TJSONString.Create(TextVal); + TextVal := JsonText.ToJSON; + JsonText.Free; + end; + Values.Add(Col.Connection.EscapeString(TextVal)); + end; + + dtcBinary: ; + + dtcTemporal: begin + TextVal := ''; + case Col.DataType.Index of + dbdtDate, dbdtTime, dbdtYear, dbdtDatetime, dbdtDatetime2, dbdtTimestamp, dbdtInterval: begin + MinLen := Trunc(VarToDateTime('1971-01-01')); + MaxLen := Trunc(VarToDateTime('2035-01-01')); + FloatVal := RandomRange(MinLen, MaxLen) + Random; + TextVal := FormatDateTime(Col.DataType.Format, FloatVal, MainForm.FormatSettings); + end; + + dbdtDatetimeOffset: ; + dbdtSmalldatetime: ; + end; + Values.Add(Col.Connection.EscapeString(TextVal)); + end; + + dtcSpatial: ; + + dtcOther: begin + case Col.DataType.Index of + dbdtEnum, dbdtSet: begin + EnumValues := Col.ValueList; + IntVal := RandomRange(0, EnumValues.Count); + TextVal := EnumValues[IntVal]; + EnumValues.Free; + Values.Add(Col.Connection.EscapeString(TextVal)); + end; + dbdtBool: begin + IntVal := RandomRange(0, 2); + TextVal := IfThen(IntVal=0, 'true', 'false'); + Values.Add(Col.Connection.EscapeString(TextVal)); + end; + else + Values.Add('0'); + end; + end; + end; + end; + InsertSql := InsertSqlBase + '(' + Implode(', ', Values) + ')'; + DBObj.Connection.Query(InsertSql, False, lcScript); + end; + + ColumnNamesQuoted.Free; + ColumnNamesSkipped.Free; + Values.Free; +end; + + procedure TfrmTableTools.CheckAllClick(Sender: TObject); var DBNode, ObjNode: PVirtualNode;