diff --git a/source/const.inc b/source/const.inc index c788930b..c5f02edd 100644 --- a/source/const.inc +++ b/source/const.inc @@ -155,12 +155,13 @@ const REGNAME_CSV_ESCAPER = 'CSVImportFieldEscaperV2'; REGNAME_EXPORT_LOCALENUMBERS = 'ExportLocaleNumberFormats'; DEFAULT_EXPORT_LOCALENUMBERS = False; + REGNAME_CSV_WINDOWWIDTH = 'CSVImportWindowWidth'; + REGNAME_CSV_WINDOWHEIGHT = 'CSVImportWindowHeight'; REGNAME_CSV_FILENAME = 'loadfilename'; REGNAME_CSV_ENCLOPTION = 'CSVImportFieldsEnclosedOptionallyV2'; REGNAME_CSV_IGNORELINES = 'CSVImportIgnoreLines'; - REGNAME_CSV_LOWPRIO = 'CSVImportLowPriority'; - REGNAME_CSV_REPLACE = 'CSVImportReplace'; - REGNAME_CSV_IGNORE = 'CSVImportIgnore'; + REGNAME_CSV_DUPLICATES = 'CSVImportDuplicateHandling'; + REGNAME_CSV_PARSEMETHOD = 'CSVImportParseMethod'; REGNAME_COPYMAXSIZE = 'CopyDataMaxSize'; DEFAULT_COPYMAXSIZE = 5; REGNAME_DO_UPDATECHECK = 'Updatecheck'; diff --git a/source/loaddata.dfm b/source/loaddata.dfm index b4a31d53..c9006f2b 100644 --- a/source/loaddata.dfm +++ b/source/loaddata.dfm @@ -1,12 +1,13 @@ object loaddataform: Tloaddataform Left = 212 Top = 111 - BorderStyle = bsDialog BorderWidth = 3 Caption = 'Import text file' - ClientHeight = 343 - ClientWidth = 423 + ClientHeight = 488 + ClientWidth = 408 Color = clBtnFace + Constraints.MinHeight = 530 + Constraints.MinWidth = 430 Font.Charset = DEFAULT_CHARSET Font.Color = clWindowText Font.Height = -11 @@ -18,13 +19,13 @@ object loaddataform: Tloaddataform OnDestroy = FormDestroy OnShow = FormShow DesignSize = ( - 423 - 343) + 408 + 488) PixelsPerInch = 96 TextHeight = 13 object btnImport: TButton - Left = 268 - Top = 316 + Left = 244 + Top = 455 Width = 75 Height = 25 Anchors = [akRight, akBottom] @@ -32,339 +33,311 @@ object loaddataform: Tloaddataform Default = True Enabled = False ModalResult = 1 - TabOrder = 0 + TabOrder = 6 OnClick = btnImportClick end object btnCancel: TButton - Left = 348 - Top = 316 + Left = 325 + Top = 455 Width = 75 Height = 25 Anchors = [akRight, akBottom] Cancel = True Caption = 'Cancel' ModalResult = 2 + TabOrder = 7 + end + object grpFilename: TGroupBox + Left = 8 + Top = 8 + Width = 392 + Height = 84 + Anchors = [akLeft, akTop, akRight] + Caption = 'Input file' + TabOrder = 0 + DesignSize = ( + 392 + 84) + object lblFilename: TLabel + Left = 10 + Top = 27 + Width = 46 + Height = 13 + Caption = 'Filename:' + FocusControl = editFilename + end + object lblEncoding: TLabel + Left = 10 + Top = 54 + Width = 47 + Height = 13 + Caption = 'Encoding:' + end + object editFilename: TButtonedEdit + Left = 88 + Top = 24 + Width = 294 + Height = 21 + Anchors = [akLeft, akTop, akRight] + Images = MainForm.ImageListMain + RightButton.ImageIndex = 51 + RightButton.Visible = True + TabOrder = 0 + Text = 'editFilename' + OnChange = editFilenameChange + OnDblClick = btnOpenFileClick + OnRightButtonClick = btnOpenFileClick + end + object comboEncoding: TComboBox + Left = 88 + Top = 51 + Width = 294 + Height = 21 + Style = csDropDownList + Anchors = [akLeft, akTop, akRight] + DropDownCount = 16 + TabOrder = 1 + OnSelect = comboEncodingSelect + end + end + object grpFields: TGroupBox + Left = 8 + Top = 93 + Width = 209 + Height = 109 + Caption = 'Fields' TabOrder = 1 + DesignSize = ( + 209 + 109) + object lblFieldTerminater: TLabel + Left = 10 + Top = 26 + Width = 67 + Height = 13 + Caption = 'terminated by' + end + object lblFieldEncloser: TLabel + Left = 10 + Top = 51 + Width = 57 + Height = 13 + Caption = 'enclosed by' + end + object lblFieldEscaper: TLabel + Left = 10 + Top = 75 + Width = 55 + Height = 13 + Caption = 'escaped by' + end + object editFieldEscaper: TEdit + Left = 88 + Top = 72 + Width = 49 + Height = 21 + TabOrder = 2 + Text = '"' + end + object editFieldEncloser: TEdit + Left = 88 + Top = 48 + Width = 49 + Height = 21 + TabOrder = 1 + Text = '"' + end + object editFieldTerminator: TEdit + Left = 88 + Top = 24 + Width = 49 + Height = 21 + TabOrder = 0 + Text = ';' + end + object chkFieldsEnclosedOptionally: TCheckBox + Left = 143 + Top = 50 + Width = 62 + Height = 17 + Anchors = [akLeft, akTop, akRight] + Caption = 'optionally' + Checked = True + State = cbChecked + TabOrder = 3 + end end - object PageControlMain: TPageControl - Left = 0 - Top = 0 - Width = 423 - Height = 310 - ActivePage = tabSource - Align = alTop - Anchors = [akLeft, akTop, akRight, akBottom] + object grpLines: TGroupBox + Left = 223 + Top = 93 + Width = 177 + Height = 109 + Anchors = [akLeft, akTop, akRight] + Caption = 'Lines' TabOrder = 2 - object tabSource: TTabSheet - Caption = 'Source' - DesignSize = ( - 415 - 282) - object grpFilename: TGroupBox - Left = 5 - Top = 2 - Width = 403 - Height = 90 - Anchors = [akLeft, akTop, akRight] - Caption = 'File' - TabOrder = 0 - DesignSize = ( - 403 - 90) - object lblFilename: TLabel - Left = 10 - Top = 27 - Width = 46 - Height = 13 - Caption = 'Filename:' - FocusControl = editFilename - end - object lblCharset: TLabel - Left = 10 - Top = 54 - Width = 70 - Height = 13 - Caption = '&Character set:' - FocusControl = comboCharset - end - object editFilename: TButtonedEdit - Left = 104 - Top = 24 - Width = 289 - Height = 21 - Anchors = [akLeft, akTop, akRight] - Images = MainForm.ImageListMain - RightButton.ImageIndex = 51 - RightButton.Visible = True - TabOrder = 0 - Text = 'editFilename' - OnChange = editFilenameChange - OnDblClick = btnOpenFileClick - OnRightButtonClick = btnOpenFileClick - end - object comboCharset: TComboBox - Left = 104 - Top = 51 - Width = 289 - Height = 21 - Style = csDropDownList - Anchors = [akLeft, akTop, akRight] - DropDownCount = 16 - TabOrder = 1 - end - end - object grpFields: TGroupBox - Left = 5 - Top = 93 - Width = 403 - Height = 109 - Anchors = [akLeft, akTop, akRight] - Caption = 'Fields' - TabOrder = 1 - object lblFieldTerminater: TLabel - Left = 10 - Top = 26 - Width = 67 - Height = 13 - Caption = 'terminated by' - end - object lblFieldEncloser: TLabel - Left = 10 - Top = 51 - Width = 57 - Height = 13 - Caption = 'enclosed by' - end - object lblFieldEscaper: TLabel - Left = 10 - Top = 75 - Width = 55 - Height = 13 - Caption = 'escaped by' - end - object editFieldEscaper: TEdit - Left = 104 - Top = 72 - Width = 49 - Height = 21 - TabOrder = 0 - Text = '"' - end - object editFieldEncloser: TEdit - Left = 104 - Top = 48 - Width = 49 - Height = 21 - TabOrder = 1 - Text = '"' - end - object editFieldTerminator: TEdit - Left = 104 - Top = 24 - Width = 49 - Height = 21 - TabOrder = 2 - Text = ';' - end - object chkFieldsEnclosedOptionally: TCheckBox - Left = 167 - Top = 50 - Width = 73 - Height = 17 - Caption = 'optionally' - Checked = True - State = cbChecked - TabOrder = 3 - end - end - object grpLines: TGroupBox - Left = 5 - Top = 202 - Width = 403 - Height = 74 - Anchors = [akLeft, akTop, akRight] - Caption = 'Lines' - TabOrder = 2 - object lblIgnoreLinesCount: TLabel - Left = 166 - Top = 44 - Width = 24 - Height = 13 - Caption = 'Lines' - end - object lblLineTerminator: TLabel - Left = 16 - Top = 20 - Width = 67 - Height = 13 - Caption = 'terminated by' - end - object lblIgnoreLines: TLabel - Left = 16 - Top = 44 - Width = 30 - Height = 13 - Caption = 'ignore' - end - object updownIgnoreLines: TUpDown - Left = 141 - Top = 41 - Width = 16 - Height = 21 - Associate = editIgnoreLines - Max = 32767 - Position = 1 - TabOrder = 0 - end - object editIgnoreLines: TEdit - Left = 108 - Top = 41 - Width = 33 - Height = 21 - TabOrder = 1 - Text = '1' - end - object editLineTerminator: TEdit - Left = 108 - Top = 17 - Width = 49 - Height = 21 - TabOrder = 2 - Text = '\r\n' - end - end + object lblIgnoreLinesCount: TLabel + Left = 143 + Top = 44 + Width = 24 + Height = 13 + Caption = 'Lines' end - object tabDestination: TTabSheet - Caption = 'Destination' - ImageIndex = 1 - DesignSize = ( - 415 - 282) - object lblDatabase: TLabel - Left = 10 - Top = 10 - Width = 50 - Height = 13 - Caption = 'Database:' - end - object lblTable: TLabel - Left = 10 - Top = 53 - Width = 84 - Height = 13 - Caption = 'Import into table:' - end - object lblColumns: TLabel - Left = 10 - Top = 101 - Width = 65 - Height = 13 - Caption = 'Use Columns:' - end - object comboDatabase: TComboBox - Left = 10 - Top = 26 - Width = 164 - Height = 21 - Style = csDropDownList - Anchors = [akLeft, akTop, akRight] - TabOrder = 0 - OnChange = comboDatabaseChange - end - object comboTable: TComboBox - Left = 10 - Top = 69 - Width = 164 - Height = 21 - Style = csDropDownList - Anchors = [akLeft, akTop, akRight] - TabOrder = 1 - OnChange = comboTableChange - end - object chklistColumns: TCheckListBox - Left = 10 - Top = 117 - Width = 133 - Height = 150 - Anchors = [akLeft, akTop, akRight, akBottom] - ItemHeight = 13 - TabOrder = 2 - end - object grpOptions: TGroupBox - Left = 196 - Top = 10 - Width = 209 - Height = 105 - Anchors = [akTop, akRight] - Caption = 'Options' - TabOrder = 3 - object lblDuplicates: TLabel - Left = 16 - Top = 54 - Width = 143 - Height = 13 - Caption = 'Handling of duplicate records:' - end - object chkLowPriority: TCheckBox - Left = 16 - Top = 23 - Width = 81 - Height = 17 - Caption = 'Low Priority' - TabOrder = 0 - end - object chkReplace: TCheckBox - Left = 16 - Top = 71 - Width = 65 - Height = 17 - Caption = 'Replace' - TabOrder = 1 - OnClick = chkReplaceClick - end - object chkIgnore: TCheckBox - Left = 96 - Top = 70 - Width = 57 - Height = 17 - Caption = 'Ignore' - TabOrder = 2 - OnClick = chkIgnoreClick - end - end - object ToolBarColMove: TToolBar - Left = 149 - Top = 117 - Width = 23 - Height = 44 - Align = alNone - AutoSize = True - Caption = 'ToolBarColMove' - Images = MainForm.ImageListMain - TabOrder = 4 - object btnColUp: TToolButton - Left = 0 - Top = 0 - Caption = 'btnColUp' - ImageIndex = 74 - Wrap = True - OnClick = btnColUpClick - end - object btnColDown: TToolButton - Left = 0 - Top = 22 - Caption = 'btnColDown' - ImageIndex = 75 - OnClick = btnColDownClick - end - end + object lblLineTerminator: TLabel + Left = 10 + Top = 20 + Width = 67 + Height = 13 + Caption = 'terminated by' + end + object lblIgnoreLines: TLabel + Left = 10 + Top = 44 + Width = 30 + Height = 13 + Caption = 'ignore' + end + object updownIgnoreLines: TUpDown + Left = 121 + Top = 41 + Width = 16 + Height = 21 + Associate = editIgnoreLines + Max = 32767 + Position = 1 + TabOrder = 2 + end + object editIgnoreLines: TEdit + Left = 88 + Top = 41 + Width = 33 + Height = 21 + TabOrder = 1 + Text = '1' + end + object editLineTerminator: TEdit + Left = 88 + Top = 17 + Width = 49 + Height = 21 + TabOrder = 0 + Text = '\r\n' end end - object OpenDialogCSVFile: TOpenDialog - DefaultExt = 'csv' - Filter = - 'MySQL CSV files (*.csv)|*.csv|Text files (*.txt)|*.txt|All files' + - ' (*.*)|*.*' - Left = 392 + object grpDuplicates: TRadioGroup + Left = 8 + Top = 208 + Width = 209 + Height = 130 + Caption = 'Handling of duplicate rows' + ItemIndex = 2 + Items.Strings = ( + 'INSERT (may throw errors)' + 'INSERT IGNORE (duplicates)' + 'REPLACE (duplicates)') + TabOrder = 3 + end + object grpParseMethod: TRadioGroup + Left = 8 + Top = 344 + Width = 209 + Height = 105 + Anchors = [akLeft, akTop, akBottom] + Caption = 'Method' + ItemIndex = 0 + Items.Strings = ( + 'Server parses file contents (LOAD DATA)' + 'Client parses file contents') + TabOrder = 4 + WordWrap = True + OnClick = grpParseMethodClick + end + object grpDestination: TGroupBox + Left = 223 + Top = 208 + Width = 177 + Height = 241 + Anchors = [akLeft, akTop, akRight, akBottom] + Caption = 'Destination' + TabOrder = 5 + DesignSize = ( + 177 + 241) + object lblDatabase: TLabel + Left = 10 + Top = 24 + Width = 50 + Height = 13 + Caption = 'Database:' + end + object lblTable: TLabel + Left = 10 + Top = 48 + Width = 30 + Height = 13 + Caption = 'Table:' + end + object lblColumns: TLabel + Left = 10 + Top = 72 + Width = 44 + Height = 13 + Caption = 'Columns:' + end + object comboDatabase: TComboBox + Left = 64 + Top = 21 + Width = 103 + Height = 21 + Style = csDropDownList + Anchors = [akLeft, akTop, akRight] + TabOrder = 0 + OnChange = comboDatabaseChange + end + object comboTable: TComboBox + Left = 64 + Top = 45 + Width = 103 + Height = 21 + Style = csDropDownList + Anchors = [akLeft, akTop, akRight] + TabOrder = 1 + OnChange = comboTableChange + end + object chklistColumns: TCheckListBox + Left = 10 + Top = 91 + Width = 128 + Height = 141 + Anchors = [akLeft, akTop, akRight, akBottom] + ItemHeight = 13 + TabOrder = 2 + end + object ToolBarColMove: TToolBar + Left = 144 + Top = 91 + Width = 23 + Height = 44 + Align = alNone + Anchors = [akTop, akRight] + AutoSize = True + Caption = 'ToolBarColMove' + Images = MainForm.ImageListMain + TabOrder = 3 + object btnColUp: TToolButton + Left = 0 + Top = 0 + Caption = 'btnColUp' + ImageIndex = 74 + Wrap = True + OnClick = btnColMoveClick + end + object btnColDown: TToolButton + Left = 0 + Top = 22 + Caption = 'btnColDown' + ImageIndex = 75 + OnClick = btnColMoveClick + end + end end end diff --git a/source/loaddata.pas b/source/loaddata.pas index e46f7368..709ff76e 100644 --- a/source/loaddata.pas +++ b/source/loaddata.pas @@ -10,17 +10,13 @@ interface uses Windows, SysUtils, Classes, Controls, Forms, Dialogs, StdCtrls, ComCtrls, CheckLst, - SynRegExpr, Buttons, ExtCtrls, ToolWin, + SynRegExpr, Buttons, ExtCtrls, ToolWin, ExtDlgs, Math, mysql_connection; type Tloaddataform = class(TForm) btnImport: TButton; btnCancel: TButton; - OpenDialogCSVFile: TOpenDialog; - PageControlMain: TPageControl; - tabSource: TTabSheet; - tabDestination: TTabSheet; lblDatabase: TLabel; comboDatabase: TComboBox; lblTable: TLabel; @@ -30,11 +26,6 @@ type ToolBarColMove: TToolBar; btnColUp: TToolButton; btnColDown: TToolButton; - grpOptions: TGroupBox; - chkLowPriority: TCheckBox; - chkReplace: TCheckBox; - chkIgnore: TCheckBox; - lblDuplicates: TLabel; grpFilename: TGroupBox; editFilename: TButtonedEdit; grpFields: TGroupBox; @@ -53,8 +44,11 @@ type lblLineTerminator: TLabel; lblIgnoreLines: TLabel; lblFilename: TLabel; - comboCharset: TComboBox; - lblCharset: TLabel; + comboEncoding: TComboBox; + lblEncoding: TLabel; + grpDuplicates: TRadioGroup; + grpParseMethod: TRadioGroup; + grpDestination: TGroupBox; procedure FormCreate(Sender: TObject); procedure FormDestroy(Sender: TObject); procedure editFilenameChange(Sender: TObject); @@ -62,13 +56,18 @@ type procedure comboDatabaseChange(Sender: TObject); procedure comboTableChange(Sender: TObject); procedure btnImportClick(Sender: TObject); + procedure ServerParse(Sender: TObject); + procedure ClientParse(Sender: TObject); procedure btnOpenFileClick(Sender: TObject); - procedure chkReplaceClick(Sender: TObject); - procedure chkIgnoreClick(Sender: TObject); - procedure btnColUpClick(Sender: TObject); - procedure btnColDownClick(Sender: TObject); + procedure btnColMoveClick(Sender: TObject); + procedure grpParseMethodClick(Sender: TObject); + procedure comboEncodingSelect(Sender: TObject); private { Private declarations } + Encoding: TEncoding; + Term, Encl, Escp, LineTerm: String; + RowCount, ColumnCount: Integer; + SelectedCharsetIndex: Integer; public { Public declarations } end; @@ -85,7 +84,10 @@ uses Main, helpers; procedure Tloaddataform.FormCreate(Sender: TObject); begin InheritFont(Font); + SetWindowSizeGrip(Handle, True); // Restore settings + Width := GetRegValue(REGNAME_CSV_WINDOWWIDTH, Width); + Height := GetRegValue(REGNAME_CSV_WINDOWHEIGHT, Height); editFilename.Text := GetRegValue(REGNAME_CSV_FILENAME, ''); editFieldTerminator.Text := GetRegValue(REGNAME_CSV_SEPARATOR, DEFAULT_CSV_SEPARATOR); editFieldEncloser.Text := GetRegValue(REGNAME_CSV_ENCLOSER, DEFAULT_CSV_ENCLOSER); @@ -93,9 +95,8 @@ begin chkFieldsEnclosedOptionally.Checked := GetRegValue(REGNAME_CSV_ENCLOPTION, chkFieldsEnclosedOptionally.Checked); editFieldEscaper.Text := GetRegValue(REGNAME_CSV_ESCAPER, editFieldEscaper.Text); updownIgnoreLines.Position := GetRegValue(REGNAME_CSV_IGNORELINES, updownIgnoreLines.Position); - chkLowPriority.Checked := GetRegValue(REGNAME_CSV_LOWPRIO, chkLowPriority.Checked); - chkReplace.Checked := GetRegValue(REGNAME_CSV_REPLACE, chkReplace.Checked); - chkIgnore.Checked := GetRegValue(REGNAME_CSV_IGNORE, chkIgnore.Checked); + grpDuplicates.ItemIndex := GetRegValue(REGNAME_CSV_DUPLICATES, grpDuplicates.ItemIndex); + grpParseMethod.ItemIndex := GetRegValue(REGNAME_CSV_PARSEMETHOD, grpParseMethod.ItemIndex); end; @@ -103,6 +104,8 @@ procedure Tloaddataform.FormDestroy(Sender: TObject); begin // Save settings OpenRegistry; + MainReg.WriteInteger(REGNAME_CSV_WINDOWWIDTH, Width); + MainReg.WriteInteger(REGNAME_CSV_WINDOWHEIGHT, Height); MainReg.WriteString(REGNAME_CSV_FILENAME, editFilename.Text); MainReg.WriteString(REGNAME_CSV_SEPARATOR, editFieldTerminator.Text); MainReg.WriteString(REGNAME_CSV_ENCLOSER, editFieldEncloser.Text); @@ -110,9 +113,8 @@ begin MainReg.WriteBool(REGNAME_CSV_ENCLOPTION, chkFieldsEnclosedOptionally.Checked); MainReg.WriteString(REGNAME_CSV_ESCAPER, editFieldEscaper.Text); MainReg.WriteInteger(REGNAME_CSV_IGNORELINES, updownIgnoreLines.Position); - MainReg.WriteBool(REGNAME_CSV_LOWPRIO, chkLowPriority.Checked); - MainReg.WriteBool(REGNAME_CSV_REPLACE, chkReplace.Checked); - MainReg.WriteBool(REGNAME_CSV_IGNORE, chkIgnore.Checked); + MainReg.WriteInteger(REGNAME_CSV_DUPLICATES, grpDuplicates.ItemIndex); + MainReg.WriteInteger(REGNAME_CSV_PARSEMETHOD, grpParseMethod.ItemIndex); end; @@ -124,18 +126,65 @@ begin comboDatabase.ItemIndex := comboDatabase.Items.IndexOf( Mainform.ActiveDatabase ); if comboDatabase.ItemIndex = -1 then comboDatabase.ItemIndex := 0; - comboDatabaseChange(self); + comboDatabaseChange(Sender); + editFilename.SetFocus; +end; + + +procedure Tloaddataform.grpParseMethodClick(Sender: TObject); +var + ServerWillParse: Boolean; + Charset, DefCharset, dbcreate: String; + v: Integer; + CharsetTable: TMySQLQuery; + rx: TRegExpr; +begin + ServerWillParse := grpParseMethod.ItemIndex = 0; + comboEncoding.Enabled := ServerWillParse; + editFieldEscaper.Enabled := ServerWillParse; + chkFieldsEnclosedOptionally.Enabled := ServerWillParse; + comboEncoding.Clear; + if comboEncoding.Enabled then begin + // Populate charset combo + v := Mainform.Connection.ServerVersionInt; + if ((v >= 50038) and (v < 50100)) or (v >= 50117) then begin + Charset := MainForm.GetCharsetByEncoding(Encoding); + // Detect db charset + DefCharset := 'Let server/database decide'; + dbcreate := Mainform.Connection.GetVar('SHOW CREATE DATABASE '+Mainform.mask(comboDatabase.Text), 1); + rx := TRegExpr.Create; + rx.ModifierG := True; + rx.Expression := 'CHARACTER SET (\w+)'; + if rx.Exec(dbcreate) then + DefCharset := DefCharset + ' ('+rx.Match[1]+')'; + comboEncoding.Items.Add(DefCharset); + CharsetTable := Mainform.Connection.CharsetTable; + CharsetTable.First; + while not CharsetTable.Eof do begin + comboEncoding.Items.Add(CharsetTable.Col(1) + ' ('+CharsetTable.Col(0)+')'); + if (SelectedCharsetIndex = -1) and (Charset = CharsetTable.Col(0)) then + SelectedCharsetIndex := comboEncoding.Items.Count-1; + CharsetTable.Next; + end; + if SelectedCharsetIndex = -1 then + SelectedCharsetIndex := 0; + comboEncoding.ItemIndex := SelectedCharsetIndex; + end else begin + comboEncoding.Items.Add('Unsupported by this server'); + comboEncoding.ItemIndex := 0; + end; + end else begin + comboEncoding.Items.Add(Mainform.GetEncodingName(Encoding)); + comboEncoding.ItemIndex := 0; + end; end; procedure Tloaddataform.comboDatabaseChange(Sender: TObject); var - count, i, selCharsetIndex, v: Integer; + count, i: Integer; DBObjects: TDBObjectList; - seldb, seltable, dbcreate: String; - rx: TRegExpr; - DefCharset: String; - CharsetTable: TMySQLQuery; + seldb, seltable: String; begin // read tables from db comboTable.Items.Clear; @@ -152,43 +201,16 @@ begin if comboTable.ItemIndex = -1 then comboTable.ItemIndex := 0; - comboTableChange(self); - - selCharsetIndex := comboCharset.ItemIndex; - comboCharset.Enabled := False; - comboCharset.Clear; - v := Mainform.Connection.ServerVersionInt; - if ((v >= 50038) and (v < 50100)) or (v >= 50117) then begin - comboCharset.Enabled := True; - // Detect db charset - DefCharset := 'Let server/database decide'; - dbcreate := Mainform.Connection.GetVar('SHOW CREATE DATABASE '+Mainform.mask(comboDatabase.Text), 1); - rx := TRegExpr.Create; - rx.ModifierG := True; - rx.Expression := 'CHARACTER SET (\w+)'; - if rx.Exec(dbcreate) then - DefCharset := DefCharset + ' ('+rx.Match[1]+')'; - comboCharset.Items.Add(DefCharset); - CharsetTable := Mainform.Connection.CharsetTable; - CharsetTable.First; - while not CharsetTable.Eof do begin - comboCharset.Items.Add(CharsetTable.Col(1) + ' ('+CharsetTable.Col(0)+')'); - if CharsetTable.Col(0) = 'utf8' then begin - i := comboCharset.Items.Count-1; - comboCharset.Items[i] := comboCharset.Items[i] + ' - '+APPNAME+' output'; - if selCharsetIndex = -1 then - selCharsetIndex := i; - end; - CharsetTable.Next; - end; - comboCharset.ItemIndex := selCharsetIndex; - end else begin - comboCharset.Items.Add('Unsupported by this server'); - comboCharset.ItemIndex := 0; - end; + grpParseMethod.OnClick(Sender); + comboTableChange(Sender); end; +procedure Tloaddataform.comboEncodingSelect(Sender: TObject); +begin + SelectedCharsetIndex := comboEncoding.ItemIndex; +end; + procedure Tloaddataform.comboTableChange(Sender: TObject); begin // fill columns: @@ -200,145 +222,309 @@ begin ToggleCheckListBox( chklistColumns, True ); // Ensure valid state of Import-Button - editFilenameChange(sender); + editFilenameChange(Sender); end; procedure Tloaddataform.btnImportClick(Sender: TObject); var - query : String; - col : TStringList; - i : Integer; - - // Correctly escape field-terminator, line-terminator or encloser - // and take care of already escaped characters like \t - // See bug 1827494 - function escOptionString( str: String ): String; - begin - Result := '''' + StringReplace(str, '''', '\''', [rfReplaceAll]) + ''''; - end; + StartTickCount: Cardinal; + i: Integer; begin + Screen.Cursor := crHourglass; + StartTickCount := GetTickCount; - query := 'LOAD DATA '; + ColumnCount := 0; + for i:=0 to chkListColumns.Items.Count-1 do begin + if chkListColumns.Checked[i] then + Inc(ColumnCount); + end; - if chkLowPriority.Checked then - query := query + 'LOW_PRIORITY '; + Term := MainForm.Connection.UnescapeString(editFieldTerminator.Text); + Encl := MainForm.Connection.UnescapeString(editFieldEncloser.Text); + LineTerm := MainForm.Connection.UnescapeString(editLineTerminator.Text); + Escp := MainForm.Connection.UnescapeString(editFieldEscaper.Text); - query := query + 'LOCAL INFILE ' + esc(editFilename.Text) + ' '; - if chkReplace.Checked then - query := query + 'REPLACE ' - else if chkIgnore.Checked then - query := query + 'IGNORE '; - query := query + 'INTO TABLE ' + Mainform.Mask(comboDatabase.Text) + '.' + Mainform.Mask(comboTable.Text) + ' '; + try + case grpParseMethod.ItemIndex of + 0: ServerParse(Sender); + 1: ClientParse(Sender); + end; + MainForm.LogSQL(FormatNumber(RowCount)+' rows imported in '+FormatNumber((GetTickcount-StartTickCount)/1000, 3)+' seconds.'); + except + on E:EDatabaseError do begin + Screen.Cursor := crDefault; + ModalResult := mrNone; + MessageDlg(E.Message, mtError, [mbOK], 0); + end; + end; - if comboCharset.ItemIndex > 0 then begin - Mainform.Connection.CharsetTable.RecNo := comboCharset.ItemIndex-1; - query := query + 'CHARACTER SET '+Mainform.Connection.CharsetTable.Col(0)+' '; + Mainform.ShowStatusMsg; + Screen.Cursor := crDefault; +end; + + +procedure Tloaddataform.ServerParse(Sender: TObject); +var + SQL: String; + i: Integer; +begin + SQL := 'LOAD DATA LOCAL INFILE ' + esc(editFilename.Text) + ' '; + case grpDuplicates.ItemIndex of + 1: SQL := SQL + 'IGNORE '; + 2: SQL := SQL + 'REPLACE '; + end; + SQL := SQL + 'INTO TABLE ' + Mainform.Mask(comboDatabase.Text) + '.' + Mainform.Mask(comboTable.Text) + ' '; + + if comboEncoding.ItemIndex > 0 then begin + Mainform.Connection.CharsetTable.RecNo := comboEncoding.ItemIndex-1; + SQL := SQL + 'CHARACTER SET '+Mainform.Connection.CharsetTable.Col(0)+' '; end; // Fields: - if (editFieldTerminator.Text <> '') or (editFieldEncloser.Text <> '') or (editFieldEscaper.Text <> '') then - query := query + 'FIELDS '; + if (Term <> '') or (Encl <> '') or (Escp <> '') then + SQL := SQL + 'FIELDS '; if editFieldTerminator.Text <> '' then - query := query + 'TERMINATED BY ' + escOptionString(editFieldTerminator.Text) + ' '; - if editFieldEncloser.Text <> '' then - begin + SQL := SQL + 'TERMINATED BY ' + esc(Term) + ' '; + if Encl <> '' then begin if chkFieldsEnclosedOptionally.Checked then - query := query + 'OPTIONALLY '; - query := query + 'ENCLOSED BY ' + escOptionString(editFieldEncloser.Text) + ' '; + SQL := SQL + 'OPTIONALLY '; + SQL := SQL + 'ENCLOSED BY ' + esc(Encl) + ' '; end; - if editFieldEscaper.Text <> '' then - query := query + 'ESCAPED BY ' + escOptionString(editFieldEscaper.Text) + ' '; + if Escp <> '' then + SQL := SQL + 'ESCAPED BY ' + esc(Escp) + ' '; // Lines: - if editLineTerminator.Text <> '' then - query := query + 'LINES TERMINATED BY ' + escOptionString(editLineTerminator.Text) + ' '; + if LineTerm <> '' then + SQL := SQL + 'LINES TERMINATED BY ' + esc(LineTerm) + ' '; if updownIgnoreLines.Position > 0 then - query := query + 'IGNORE ' + inttostr(updownIgnoreLines.Position) + ' LINES '; + SQL := SQL + 'IGNORE ' + inttostr(updownIgnoreLines.Position) + ' LINES '; - col := TStringList.Create; - for i:=0 to chklistColumns.Items.Count - 1 do - begin - if chklistColumns.checked[i] then - col.Add(Mainform.Mask( chklistColumns.Items[i] )); + // Column listing + SQL := SQL + '('; + for i:=0 to chklistColumns.Items.Count-1 do begin + if chklistColumns.Checked[i] then + SQL := SQL + Mainform.Mask(chklistColumns.Items[i]) + ', '; end; + SetLength(SQL, Length(SQL)-2); + SQL := SQL + ')'; -// if col.Count < ColumnsCheckListBox.Items.Count then - query := query + '(' + implodestr(',', col) + ')'; + Mainform.Connection.Query(SQL); + RowCount := Max(MainForm.Connection.RowsAffected, 0); +end; - try - Mainform.Connection.Query(query); - except - on E:EDatabaseError do begin - MessageDlg(E.Message, mtError, [mbOk], 0); - ModalResult := mrNone; + +procedure Tloaddataform.ClientParse(Sender: TObject); +var + P, ContentLen, ProgressCharsPerStep, ProgressChars: Integer; + IgnoreLines, ValueCount, PacketSize: Integer; + EnclLen, TermLen, LineTermLen: Integer; + Contents: String; + EnclTest, TermTest, LineTermTest: String; + Value, SQL: String; + IsEncl, IsTerm, IsLineTerm: Boolean; + InEncl: Boolean; + OutStream: TMemoryStream; +const + ProgressBarSteps=100; + + procedure NextChar; + begin + Inc(P); + Inc(ProgressChars); + if ProgressChars >= ProgressCharsPerStep then begin + Mainform.ProgressBarStatus.StepIt; + Mainform.ShowStatusMsg('Importing textfile, row '+FormatNumber(RowCount-IgnoreLines)+', '+IntToStr(Mainform.ProgressBarStatus.Position)+'%'); + ProgressChars := 0; end; end; + + function TestLeftChars(var Portion: String; CompareTo: String; Len: Integer): Boolean; + var i: Integer; + begin + if Len > 0 then begin + for i:=1 to Len-1 do + Portion[i] := Portion[i+1]; + Portion[Len] := Contents[P]; + Result := Portion = CompareTo; + end else + Result := False; + end; + + procedure AddValue; + var + i: Integer; + begin + Inc(ValueCount); + if ValueCount <= ColumnCount then begin + if Copy(Value, 1, EnclLen) = Encl then begin + Delete(Value, 1, EnclLen); + Delete(Value, Length(Value)-EnclLen+1, EnclLen); + end; + if SQL = '' then begin + case grpDuplicates.ItemIndex of + 0: SQL := 'INSERT'; + 1: SQL := 'INSERT IGNORE'; + 2: SQL := 'REPLACE'; + end; + SQL := SQL + ' INTO '+MainForm.mask(comboDatabase.Text)+'.'+MainForm.mask(comboTable.Text)+' ('; + for i:=0 to chkListColumns.Items.Count-1 do begin + if chkListColumns.Checked[i] then + SQL := SQL + MainForm.mask(chkListColumns.Items[i]) + ', '; + end; + SetLength(SQL, Length(SQL)-2); + SQL := SQL + ') VALUES ('; + end; + if Value <> 'NULL' then + Value := esc(Value); + SQL := SQL + Value + ', '; + end; + Value := ''; + end; + + procedure AddRow; + var + SA: AnsiString; + ChunkSize: Int64; + i: Integer; + begin + if SQL = '' then + Exit; + Inc(RowCount); + for i:=ValueCount to ColumnCount do begin + Value := 'NULL'; + AddValue; + end; + ValueCount := 0; + if RowCount > IgnoreLines then begin + Delete(SQL, Length(SQL)-1, 2); + StreamWrite(OutStream, SQL + ')'); + SQL := ''; + if (OutStream.Size < PacketSize) and (P < ContentLen) then + SQL := SQL + ', (' + else begin + OutStream.Position := 0; + ChunkSize := OutStream.Size; + SetLength(SA, ChunkSize div SizeOf(AnsiChar)); + OutStream.Read(PAnsiChar(SA)^, ChunkSize); + OutStream.Size := 0; + Mainform.Connection.Query(UTF8ToString(SA)); + SQL := ''; + end; + end else + SQL := ''; + end; + +begin + EnableProgressBar(ProgressBarSteps); + + TermLen := Length(Term); + EnclLen := Length(Encl); + LineTermLen := Length(LineTerm); + + SetLength(TermTest, TermLen); + SetLength(EnclTest, EnclLen); + SetLength(LineTermTest, LineTermLen); + + InEncl := False; + + SQL := ''; + Value := ''; + OutStream := TMemoryStream.Create; + + MainForm.ShowStatusMsg('Reading textfile ('+FormatByteNumber(_GetFileSize(editFilename.Text))+') ...'); + Contents := ReadTextfile(editFilename.Text, Encoding); + ContentLen := Length(Contents); + MainForm.ShowStatusMsg; + + P := 0; + ProgressCharsPerStep := ContentLen div ProgressBarSteps; + ProgressChars := 0; + RowCount := 0; + IgnoreLines := UpDownIgnoreLines.Position; + ValueCount := 0; + PacketSize := SIZE_MB div 2; + NextChar; + + // TODO: read chunks! + while P <= ContentLen do begin + // Check characters left-side from current position + IsEncl := TestLeftChars(EnclTest, Encl, EnclLen); + IsTerm := TestLeftChars(TermTest, Term, TermLen); + IsLineTerm := TestLeftChars(LineTermTest, LineTerm, LineTermLen) and (ValueCount >= ColumnCount-1); + + Value := Value + Contents[P]; + + if IsEncl then + InEncl := not InEncl; + + if not InEncl then begin + if IsTerm then begin + SetLength(Value, Length(Value)-TermLen); + AddValue; + end else if IsLineTerm then begin + SetLength(Value, Length(Value)-LineTermLen); + AddValue; + end; + end; + + if IsLineTerm and (not InEncl) then + AddRow; + + NextChar; + end; + // Will check if SQL is empty and not run any query in that case: + AddRow; + + Contents := ''; + FreeAndNil(OutStream); + RowCount := Max(RowCount-IgnoreLines, 0); + Mainform.ProgressBarStatus.Hide; end; + procedure Tloaddataform.btnOpenFileClick(Sender: TObject); -begin - if OpenDialogCSVFile.Execute then - editfilename.Text := OpenDialogCSVFile.FileName; -end; - -procedure Tloaddataform.chkReplaceClick(Sender: TObject); -begin - if chkReplace.Checked then - chkIgnore.checked := false; -end; - -procedure Tloaddataform.chkIgnoreClick(Sender: TObject); -begin - if chkIgnore.Checked then - chkReplace.checked := false; -end; - -procedure Tloaddataform.btnColUpClick(Sender: TObject); var - strtemp : String; - strchecked : boolean; + Dialog: TOpenTextFileDialog; + TestStream: TFileStream; begin - // move item up! - if chklistColumns.ItemIndex > -1 then - begin - if chklistColumns.ItemIndex > 0 then - begin // not first item... - strtemp := chklistColumns.Items[chklistColumns.ItemIndex-1]; - strchecked := chklistColumns.Checked[chklistColumns.ItemIndex-1]; - // replace old with new item... - chklistColumns.Items[chklistColumns.ItemIndex-1] := chklistColumns.Items[chklistColumns.ItemIndex]; - chklistColumns.Checked[chklistColumns.ItemIndex-1] := chklistColumns.Checked[chklistColumns.ItemIndex]; - // and set old item to its origin values... - chklistColumns.Items[chklistColumns.ItemIndex] := strtemp; - chklistColumns.Checked[chklistColumns.ItemIndex] := strchecked; - - chklistColumns.ItemIndex := chklistColumns.ItemIndex-1; + Dialog := TOpenTextFileDialog.Create(Self); + Dialog.Filter := 'MySQL CSV files (*.csv)|*.csv|Text files (*.txt)|*.txt|All files (*.*)|*.*'; + Dialog.DefaultExt := 'csv'; + Dialog.Encodings.Assign(Mainform.FileEncodings); + Dialog.EncodingIndex := 0; + if Dialog.Execute then begin + editfilename.Text := Dialog.FileName; + Encoding := Mainform.GetEncodingByName(Dialog.Encodings[Dialog.EncodingIndex]); + if Encoding = nil then begin + TestStream := TFileStream.Create(Dialog.Filename, fmOpenRead or fmShareDenyNone); + Encoding := DetectEncoding(TestStream); + TestStream.Free; end; + SelectedCharsetIndex := -1; + grpParseMethod.OnClick(Sender); end; + Dialog.Free; end; -procedure Tloaddataform.btnColDownClick(Sender: TObject); -var - strtemp : String; - strchecked : boolean; -begin - // move item down! - if chklistColumns.ItemIndex > -1 then - begin - if chklistColumns.ItemIndex < chklistColumns.Items.count-1 then - begin // not last item... - strtemp := chklistColumns.Items[chklistColumns.ItemIndex+1]; - strchecked := chklistColumns.Checked[chklistColumns.ItemIndex+1]; - // replace old with new item... - chklistColumns.Items[chklistColumns.ItemIndex+1] := chklistColumns.Items[chklistColumns.ItemIndex]; - chklistColumns.Checked[chklistColumns.ItemIndex+1] := chklistColumns.Checked[chklistColumns.ItemIndex]; - // and set old item to its origin values... - chklistColumns.Items[chklistColumns.ItemIndex] := strtemp; - chklistColumns.Checked[chklistColumns.ItemIndex] := strchecked; - chklistColumns.ItemIndex := chklistColumns.ItemIndex+1; - end; +procedure Tloaddataform.btnColMoveClick(Sender: TObject); +var + CheckedSelected, CheckedTarget: Boolean; + TargetIndex: Integer; +begin + // Move column name and its checkstate up or down + if Sender = btnColUp then + TargetIndex := chklistColumns.ItemIndex-1 + else + TargetIndex := chklistColumns.ItemIndex+1; + if (TargetIndex > -1) and (TargetIndex < chklistColumns.Count) then begin + CheckedSelected := chklistColumns.Checked[chklistColumns.ItemIndex]; + CheckedTarget := chklistColumns.Checked[TargetIndex]; + chklistColumns.Items.Exchange(chklistColumns.ItemIndex, TargetIndex); + chklistColumns.Checked[chklistColumns.ItemIndex] := CheckedTarget; + chklistColumns.Checked[TargetIndex] := CheckedSelected; + chklistColumns.ItemIndex := TargetIndex; end; end; diff --git a/source/main.pas b/source/main.pas index d32bf1d2..44b070b6 100644 --- a/source/main.pas +++ b/source/main.pas @@ -992,6 +992,8 @@ type function AnyGridEnsureFullRow(Grid: TVirtualStringTree; Node: PVirtualNode): Boolean; procedure DataGridEnsureFullRows(Grid: TVirtualStringTree; SelectedOnly: Boolean); function GetEncodingByName(Name: String): TEncoding; + function GetEncodingName(Encoding: TEncoding): String; + function GetCharsetByEncoding(Encoding: TEncoding): String; end; @@ -1160,6 +1162,7 @@ begin FreeAndNil(CreateDatabaseForm); FreeAndNil(SearchReplaceDialog); FreeAndNil(CopyTableDialog); + FreeAndNil(ImportTextfileDialog); // Close database connection DoDisconnect; @@ -9296,6 +9299,90 @@ begin end; +function TMainForm.GetEncodingName(Encoding: TEncoding): String; +var + idx: Integer; +begin + if Encoding = TEncoding.Default then idx := 1 + else if Encoding = TEncoding.ASCII then idx := 2 + else if Encoding = TEncoding.Unicode then idx := 3 + else if Encoding = TEncoding.BigEndianUnicode then idx := 4 + else if Encoding = TEncoding.UTF8 then idx := 5 + else if Encoding = TEncoding.UTF7 then idx := 6 + else idx := 0; + Result := FileEncodings[idx]; +end; + + +function TMainForm.GetCharsetByEncoding(Encoding: TEncoding): String; +begin + Result := ''; + if Encoding = TEncoding.Default then begin + // Listing taken from http://forge.mysql.com/worklog/task.php?id=1349 + case GetACP of + 437: Result := 'cp850'; + 850: Result := 'cp850'; + 852: Result := 'cp852'; + 858: Result := 'cp850'; + 866: Result := 'cp866'; + 874: Result := 'tis620'; + 932: Result := 'cp932'; + 936: Result := 'gbk'; + 949: Result := 'euckr'; + 959: Result := 'big5'; + 1200: Result := 'utf16le'; + 1201: Result := 'utf16'; + 1250: Result := 'latin2'; + 1251: Result := 'cp1251'; + 1252: Result := 'latin1'; + 1253: Result := 'greek'; + 1254: Result := 'latin5'; + 1255: Result := 'hebrew'; + 1256: Result := 'cp1256'; + 1257: Result := 'cp1257'; + 10000: Result := 'macroman'; + 10001: Result := 'sjis'; + 10002: Result := 'big5'; + 10008: Result := 'gb2312'; + 10021: Result := 'tis620'; + 10029: Result := 'macce'; + 12001: Result := 'utf32'; + 20107: Result := 'swe7'; + 20127: Result := 'ascii'; + 20866: Result := 'koi8r'; + 20932: Result := 'ujis'; + 20936: Result := 'gb2312'; + 20949: Result := 'euckr'; + 21866: Result := 'koi8u'; + 28591: Result := 'latin1'; + 28592: Result := 'latin2'; + 28597: Result := 'greek'; + 28598: Result := 'hebrew'; + 28599: Result := 'latin5'; + 28603: Result := 'latin7'; + 28605: Result := 'latin9'; + 38598: Result := 'hebrew'; + 51932: Result := 'ujis'; + 51936: Result := 'gb2312'; + 51949: Result := 'euckr'; + 51950: Result := 'big5'; + 54936: Result := 'gb18030'; + 65001: Result := 'utf8'; + end; + end else if Encoding = TEncoding.ASCII then + Result := 'ascii' + else if Encoding = TEncoding.Unicode then + Result := 'utf16le' + else if Encoding = TEncoding.BigEndianUnicode then + Result := 'utf16' + else if Encoding = TEncoding.UTF8 then + Result := 'utf8' + else if Encoding = TEncoding.UTF7 then + Result := 'utf7'; + // Auto-detection not supported here +end; + + procedure TMainForm.treeQueryHelpersBeforeCellPaint(Sender: TBaseVirtualTree; TargetCanvas: TCanvas; Node: PVirtualNode; Column: TColumnIndex; CellPaintMode: TVTCellPaintMode; CellRect: TRect; var ContentRect: TRect);