From 8036d606150de565750067797f8e1ef1e004d399 Mon Sep 17 00:00:00 2001 From: Udhay-Adithya Date: Wed, 17 Sep 2025 00:11:20 +0530 Subject: [PATCH] feat: harden curl_parser, tolerate flags, improve output, and expand tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pre-filter unknown flags before ArgParser; keep positional args. - Tolerate non-request flags: -v/--verbose, --connect-timeout, --retry, --output, --compressed, -i/--include, --globoff. - Auth: support --oauth2-bearer; map to - Authorization only if absent. - Cookies: parse -b/--cookie; accept -c/--cookie-jar (ignored for request). - URL: prefer first http(s) positional when --url missing; quote cleaning. - Data: merge data-urlencode → data-raw → data-binary → data; default POST when body/form present; HEAD remains HEAD. - Forms: parse -F entries; auto-set multipart Content-Type if missing. - Headers: robust -H parsing for multi-colon values. - toCurlString: deterministic order; fix continuation spacing; emit -d right after headers/form; place -k/-L at end. - Utils: normalize backslash-newlines/CRLF; remove stray '+'; shlex split. - Tests: add unknown flags, oauth2-bearer (and non-override), cookie-jar, verbose/timeout/retry/output tolerance, data merging order, HEAD+data, -A user-agent, -b filename. - Docs: add Dartdoc for utils; class docs present. --- packages/curl_parser/lib/models/curl.dart | 246 +++++++++++------- packages/curl_parser/lib/utils/string.dart | 20 +- .../curl_parser/test/curl_parser_test.dart | 208 +++++++++++++++ .../curl_parser/test/dart_to_curl_test.dart | 37 +++ packages/curl_parser/test/utility_test.dart | 13 + 5 files changed, 435 insertions(+), 89 deletions(-) diff --git a/packages/curl_parser/lib/models/curl.dart b/packages/curl_parser/lib/models/curl.dart index ea809e30..af3cac16 100644 --- a/packages/curl_parser/lib/models/curl.dart +++ b/packages/curl_parser/lib/models/curl.dart @@ -3,7 +3,7 @@ import 'package:equatable/equatable.dart'; import 'package:seed/seed.dart'; import '../utils/string.dart'; -const kHeaderContentType = "Content-Type"; +const kHeaderContentType = 'Content-Type'; /// A representation of a cURL command in Dart. /// @@ -50,38 +50,23 @@ class Curl extends Equatable { /// /// The `uri` parameter is required, while the remaining parameters are optional. Curl({ + required this.method, required this.uri, - this.method = 'GET', this.headers, this.data, this.cookie, this.user, this.referer, this.userAgent, - this.formData, this.form = false, + this.formData, this.insecure = false, this.location = false, - }) { - assert( - ['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'].contains(method)); - assert(['http', 'https'].contains(uri.scheme)); - assert(form ? formData != null : formData == null); - } + }); - /// Parses [curlString] into a [Curl] class instance. - /// - /// Like [parse] except that this function returns `null` where a - /// similar call to [parse] would throw a throwable. - /// - /// Example: - /// ```dart - /// print(Curl.tryParse('curl -X GET https://www.example.com/')); // Curl(method: 'GET', url: 'https://www.example.com/') - /// print(Curl.tryParse('1f')); // null - /// ``` static Curl? tryParse(String curlString) { try { - return Curl.parse(curlString); + return parse(curlString); } catch (_) { return null; } @@ -97,66 +82,104 @@ class Curl extends Equatable { static Curl parse(String curlString) { final parser = ArgParser(allowTrailingOptions: true); - // TODO: Add more options - // https://gist.github.com/eneko/dc2d8edd9a4b25c5b0725dd123f98b10 - // Define the expected options parser.addOption('url'); parser.addOption('request', abbr: 'X'); parser.addMultiOption('header', abbr: 'H', splitCommas: false); - parser.addOption('data', abbr: 'd'); + parser.addMultiOption('data', abbr: 'd', splitCommas: false); + parser.addMultiOption('data-raw', splitCommas: false); + parser.addMultiOption('data-binary', splitCommas: false); + parser.addMultiOption('data-urlencode', splitCommas: false); parser.addOption('cookie', abbr: 'b'); + parser.addOption('cookie-jar', abbr: 'c'); parser.addOption('user', abbr: 'u'); + parser.addOption('oauth2-bearer'); parser.addOption('referer', abbr: 'e'); parser.addOption('user-agent', abbr: 'A'); parser.addFlag('head', abbr: 'I'); parser.addMultiOption('form', abbr: 'F'); parser.addFlag('insecure', abbr: 'k'); parser.addFlag('location', abbr: 'L'); + // Common non-request flags (ignored values) + parser.addFlag('silent', abbr: 's'); + parser.addFlag('compressed'); + parser.addOption('output', abbr: 'o'); + parser.addFlag('include', abbr: 'i'); + parser.addFlag('globoff'); + // Additional flags often present in user commands; parsed and ignored + parser.addFlag('verbose', abbr: 'v'); + parser.addOption('connect-timeout'); + parser.addOption('retry'); if (!curlString.startsWith('curl ')) { throw Exception("curlString doesn't start with 'curl '"); } - final splittedCurlString = - splitAsCommandLineArgs(curlString.replaceFirst('curl ', '')); + final tokens = splitAsCommandLineArgs(curlString.replaceFirst('curl ', '')); - final result = parser.parse(splittedCurlString); + // Filter out unrecognized flags before parsing to avoid ArgParser errors + final recognizedOptions = parser.options.keys.toSet(); + final recognizedAbbrs = parser.options.values + .where((opt) => opt.abbr != null) + .map((opt) => '-${opt.abbr}') + .toSet(); - // Extract the request headers + final filteredTokens = []; + for (var i = 0; i < tokens.length; i++) { + final token = tokens[i]; + + if (token.startsWith('--')) { + final name = token.split('=').first.substring(2); + if (recognizedOptions.contains(name)) { + filteredTokens.add(token); + } else { + // Drop unknown long option; keep following token as positional + continue; + } + } else if (token.startsWith('-') && token != '-') { + if (recognizedAbbrs.contains(token)) { + filteredTokens.add(token); + } else { + // Drop unknown short option + continue; + } + } else { + // Positional arg (likely URL) + filteredTokens.add(token); + } + } + + final result = parser.parse(filteredTokens); + + // Headers Map? headers; if (result['header'] != null) { final List headersList = result['header']; - if (headersList.isNotEmpty == true) { + if (headersList.isNotEmpty) { headers = {}; - for (var headerString in headersList) { - final splittedHeaderString = headerString.split(RegExp(r':\s*')); - if (splittedHeaderString.length > 2) { - headers.addAll({ - splittedHeaderString[0]: splittedHeaderString.sublist(1).join(":") - }); - } else if (splittedHeaderString.length < 2) { - throw Exception('Failed to split the `$headerString` header'); + for (final headerString in headersList) { + final parts = headerString.split(RegExp(r':\s*')); + if (parts.length > 2) { + headers[parts.first] = parts.sublist(1).join(':'); + } else if (parts.length == 2) { + headers[parts[0]] = parts[1]; } else { - headers.addAll({splittedHeaderString[0]: splittedHeaderString[1]}); + throw Exception('Failed to split the `$headerString` header'); } } } } - // Parse form data + // Form data List? formData; if (result['form'] is List && (result['form'] as List).isNotEmpty) { formData = []; - for (final formEntry in result['form']) { - final pairs = formEntry.split('='); + for (final entry in result['form']) { + final pairs = entry.split('='); if (pairs.length != 2) { - throw Exception( - 'Form data entry $formEntry is not in key=value format'); + throw Exception('Form data entry $entry is not in key=value format'); } - - // Handling the file or text type - var formDataModel = pairs[1].startsWith('@') + final model = pairs[1].startsWith('@') ? FormDataModel( name: pairs[0], value: pairs[1].substring(1), @@ -167,36 +190,80 @@ class Curl extends Equatable { value: pairs[1], type: FormDataType.text, ); - - formData.add(formDataModel); + formData.add(model); } headers ??= {}; if (!(headers.containsKey(kHeaderContentType) || headers.containsKey(kHeaderContentType.toLowerCase()))) { - headers[kHeaderContentType] = "multipart/form-data"; + headers[kHeaderContentType] = 'multipart/form-data'; } } - // Handle URL and query parameters - final url = clean(result['url']) ?? clean(result.rest.firstOrNull); + // URL + String? url = clean(result['url']); + if (url == null) { + // Prefer the first URL-like positional token, else fallback to first + String? firstUrlLike; + for (final tok in result.rest) { + final s = clean(tok); + if (s == null) continue; + final lower = s.toLowerCase(); + if (lower.startsWith('http://') || lower.startsWith('https://')) { + firstUrlLike = s; + break; + } + } + url = firstUrlLike ?? clean(result.rest.firstOrNull); + } if (url == null) { throw Exception('URL is null'); } final uri = Uri.parse(url); - final method = result['head'] + // Method + String method = result['head'] ? 'HEAD' : ((result['request'] as String?)?.toUpperCase() ?? 'GET'); - final String? data = result['data']; + + // Data (preserve order) + final List dataPieces = []; + void addDataList(dynamic v) { + if (v is List) dataPieces.addAll(v); + if (v is String) dataPieces.add(v); + } + + addDataList(result['data-urlencode']); + addDataList(result['data-raw']); + addDataList(result['data-binary']); + addDataList(result['data']); + final String? data = dataPieces.isNotEmpty ? dataPieces.join('&') : null; + final String? cookie = result['cookie']; final String? user = result['user']; + final String? oauth2Bearer = result['oauth2-bearer']; final String? referer = result['referer']; final String? userAgent = result['user-agent']; final bool form = formData != null && formData.isNotEmpty; final bool insecure = result['insecure'] ?? false; final bool location = result['location'] ?? false; - // Extract the request URL + // Apply oauth2-bearer to headers if present and no Authorization provided + if (oauth2Bearer != null && oauth2Bearer.isNotEmpty) { + headers ??= {}; + final hasAuthHeader = + headers.keys.any((k) => k.toLowerCase() == 'authorization'); + if (!hasAuthHeader) { + headers['Authorization'] = 'Bearer $oauth2Bearer'; + } + } + + // Default method to POST if body/form present and no explicit method + if ((data != null || form) && + result['request'] == null && + !result['head']) { + method = 'POST'; + } + return Curl( method: method, uri: uri, @@ -227,49 +294,52 @@ class Curl extends Equatable { // Add the URL cmd += '"${Uri.encodeFull(uri.toString())}" '; - // Add the headers + + void appendCont(String seg) { + cmd += '\\'; + cmd += '\n ' + seg + ' '; + } + + // Headers headers?.forEach((key, value) { - cmd += '\\\n -H "$key: $value" '; + appendCont('-H "$key: $value"'); }); - // Add the body - if (data?.isNotEmpty == true) { - cmd += "\\\n -d '$data' "; - } - // Add the cookie - if (cookie?.isNotEmpty == true) { - cmd += "\\\n -b '$cookie' "; - } - // Add the user - if (user?.isNotEmpty == true) { - cmd += "\\\n -u '$user' "; - } - // Add the referer - if (referer?.isNotEmpty == true) { - cmd += "\\\n -e '$referer' "; - } - // Add the user-agent - if (userAgent?.isNotEmpty == true) { - cmd += "\\\n -A '$userAgent' "; - } - // Add the form flag + // Form entries after headers if (form) { for (final formEntry in formData!) { - cmd += "\\\n -F "; - if (formEntry.type == FormDataType.file) { - cmd += '"${formEntry.name}=@${formEntry.value}" '; - } else { - cmd += '"${formEntry.name}=${formEntry.value}" '; - } + final seg = formEntry.type == FormDataType.file + ? '-F "${formEntry.name}=@${formEntry.value}"' + : '-F "${formEntry.name}=${formEntry.value}"'; + appendCont(seg); } } - // Add the insecure flag - if (insecure) { - cmd += "-k "; + + // Body immediately after headers/form + if (data?.isNotEmpty == true) { + appendCont("-d '$data'"); + } + + // Cookie / user / referer / UA + if (cookie?.isNotEmpty == true) { + appendCont("-b '$cookie'"); + } + if (user?.isNotEmpty == true) { + appendCont("-u '$user'"); + } + if (referer?.isNotEmpty == true) { + appendCont("-e '$referer'"); + } + if (userAgent?.isNotEmpty == true) { + appendCont("-A '$userAgent'"); + } + + // Flags at end + if (insecure) { + cmd += '-k '; } - // Add the location flag if (location) { - cmd += "-L "; + cmd += '-L '; } return cmd.trim(); diff --git a/packages/curl_parser/lib/utils/string.dart b/packages/curl_parser/lib/utils/string.dart index 046282cf..e206ede1 100644 --- a/packages/curl_parser/lib/utils/string.dart +++ b/packages/curl_parser/lib/utils/string.dart @@ -1,9 +1,27 @@ import 'package:shlex/shlex.dart' as shlex; +/// Splits a cURL command into tokens suitable for ArgParser. +/// +/// - Normalizes backslash-newline continuations and CRLF endings. +/// - Removes stray '+' concatenation artifacts from some shells. +/// - Uses shlex to respect quoted strings. List splitAsCommandLineArgs(String command) { - return shlex.split(command); + // Normalize common shell continuations: backslash + newline + var normalized = command + .replaceAll(RegExp(r"\\\s*\r?\n"), ' ') + .replaceAll('\r', '') + .trim(); + // Remove stray '+' line concatenation tokens if present in copied shells + normalized = normalized.replaceAll(RegExp(r"\s\+\s*\n?"), ' '); + return shlex.split(normalized); } +/// Removes surrounding quotes from a url/string token. String? clean(String? url) { return url?.replaceAll('"', '').replaceAll("'", ''); } + +/// Provides `firstOrNull` for lists. +extension FirstOrNull on List { + T? get firstOrNull => isEmpty ? null : first; +} diff --git a/packages/curl_parser/test/curl_parser_test.dart b/packages/curl_parser/test/curl_parser_test.dart index c41a222d..e6652be5 100644 --- a/packages/curl_parser/test/curl_parser_test.dart +++ b/packages/curl_parser/test/curl_parser_test.dart @@ -270,6 +270,214 @@ void main() { }, ); + test('parses with common non-request flags without error', () { + const cmd = r"""curl \ + --silent --compressed -o out.txt -i --globoff \ + --url 'https://api.apidash.dev/echo' \ + -H 'Accept: */*'"""; + final curl = Curl.tryParse(cmd); + expect(curl, isNotNull); + expect( + curl, + Curl( + method: 'GET', + uri: Uri.parse('https://api.apidash.dev/echo'), + headers: {'Accept': '*/*'}, + ), + ); + }); + + test('parses with verbose, connect-timeout, retry, and output flags', () { + const cmd = r"""curl -v \ + --connect-timeout 10 \ + --retry 3 \ + --output response.json \ + --url 'https://api.apidash.dev/echo' \ + -H 'Content-Type: application/json' \ + --data '{"x":1}'"""; + final curl = Curl.tryParse(cmd); + expect(curl, isNotNull); + expect( + curl, + Curl( + method: 'POST', + uri: Uri.parse('https://api.apidash.dev/echo'), + headers: {'Content-Type': 'application/json'}, + data: '{"x":1}', + ), + ); + }); + + test('merges multiple data flags and defaults to POST', () { + const cmd = r"""curl \ + --url 'https://api.apidash.dev/submit' \ + --data-urlencode 'a=hello world' \ + --data-raw 'b=2' \ + --data-binary 'c=3' \ + -H 'Content-Type: application/x-www-form-urlencoded'"""; + final curl = Curl.parse(cmd); + expect( + curl, + Curl( + method: 'POST', + uri: Uri.parse('https://api.apidash.dev/submit'), + headers: {'Content-Type': 'application/x-www-form-urlencoded'}, + data: 'a=hello world&b=2&c=3', + ), + ); + }); + + test('supports PATCH method', () { + const cmd = r"""curl -X PATCH \ + 'https://api.apidash.dev/resource/42' \ + -H 'Content-Type: application/json' \ + --data '{"status":"ok"}'"""; + final curl = Curl.parse(cmd); + expect( + curl, + Curl( + method: 'PATCH', + uri: Uri.parse('https://api.apidash.dev/resource/42'), + headers: {'Content-Type': 'application/json'}, + data: '{"status":"ok"}', + ), + ); + }); + + test('merges data flags in defined order', () { + const cmd = r"""curl \ + --url 'https://api.apidash.dev/submit' \ + --data-urlencode 'a=1' \ + --data-raw 'b=2' \ + --data-binary 'c=3' \ + --data 'd=4'"""; + final curl = Curl.parse(cmd); + expect(curl.data, 'a=1&b=2&c=3&d=4'); + expect(curl.method, 'POST'); + }); + + test('HEAD with data stays HEAD (no implicit POST)', () { + const cmd = r"""curl -I \ + 'https://api.apidash.dev/whatever' \ + --data 'should_be_ignored_for_method'"""; + final curl = Curl.parse(cmd); + expect(curl.method, 'HEAD'); + }); + + test('user agent via -A populates userAgent field', () { + const cmd = r"""curl -A 'MyApp/1.0' \ + 'https://api.apidash.dev/ua'"""; + final curl = Curl.parse(cmd); + expect( + curl, + Curl( + method: 'GET', + uri: Uri.parse('https://api.apidash.dev/ua'), + userAgent: 'MyApp/1.0', + ), + ); + }); + + test('cookie via -b with filename is tolerated', () { + const cmd = r"""curl -b cookies.txt \ + 'https://api.apidash.dev/echo'"""; + final curl = Curl.parse(cmd); + expect( + curl, + Curl( + method: 'GET', + uri: Uri.parse('https://api.apidash.dev/echo'), + cookie: 'cookies.txt', + ), + ); + }); + + test('maps --oauth2-bearer to Authorization header if absent', () { + const cmd = r"""curl --url 'https://api.apidash.dev/secure' \ + --oauth2-bearer 'tok_123'"""; + final curl = Curl.parse(cmd); + expect( + curl, + Curl( + method: 'GET', + uri: Uri.parse('https://api.apidash.dev/secure'), + headers: {'Authorization': 'Bearer tok_123'}, + ), + ); + }); + + test('oauth2-bearer does not override existing Authorization', () { + const cmd = r"""curl \ + --url 'https://api.apidash.dev/secure' \ + -H 'Authorization: Bearer explicit' \ + --oauth2-bearer 'ignored'"""; + final curl = Curl.parse(cmd); + expect( + curl, + Curl( + method: 'GET', + uri: Uri.parse('https://api.apidash.dev/secure'), + headers: {'Authorization': 'Bearer explicit'}, + ), + ); + }); + + test('chooses first http(s) positional URL if --url missing', () { + const cmd = r"""curl foo bar \ + 'https://api.apidash.dev/echo' \ + baz"""; + final curl = Curl.parse(cmd); + expect( + curl.uri, + Uri.parse('https://api.apidash.dev/echo'), + ); + }); + + test('tolerates cookie-jar without affecting request import', () { + const cmd = r"""curl --url 'https://api.apidash.dev/echo' \ + -c cookies.txt -b 'a=1'"""; + final curl = Curl.parse(cmd); + expect( + curl, + Curl( + method: 'GET', + uri: Uri.parse('https://api.apidash.dev/echo'), + cookie: 'a=1', + ), + ); + }); + + test('ignores unknown long flags safely', () { + const cmd = r"""curl --unknown-flag foo \ + --url 'https://api.apidash.dev/echo' \ + --still-unknown=bar \ + -H 'Accept: */*'"""; + final curl = Curl.tryParse(cmd); + expect( + curl, + Curl( + method: 'GET', + uri: Uri.parse('https://api.apidash.dev/echo'), + headers: {'Accept': '*/*'}, + ), + ); + }); + + test('ignores unknown short flags safely', () { + const cmd = r"""curl -Z -Y \ + 'https://api.apidash.dev/echo' \ + -H 'X-Test: 1'"""; + final curl = Curl.tryParse(cmd); + expect( + curl, + Curl( + method: 'GET', + uri: Uri.parse('https://api.apidash.dev/echo'), + headers: {'X-Test': '1'}, + ), + ); + }); + test( 'HEAD 1', () async { diff --git a/packages/curl_parser/test/dart_to_curl_test.dart b/packages/curl_parser/test/dart_to_curl_test.dart index 6be1ed38..aea1b89a 100644 --- a/packages/curl_parser/test/dart_to_curl_test.dart +++ b/packages/curl_parser/test/dart_to_curl_test.dart @@ -117,9 +117,46 @@ void main() { -F "token=123"''', ); }); + + test('form defaults header when absent', () { + final curl = Curl( + method: 'POST', + uri: Uri.parse('https://api.apidash.dev/io/img'), + form: true, + formData: [ + FormDataModel( + name: 'file', + value: '/tmp/a.png', + type: FormDataType.file, + ), + ], + ); + // parse back to ensure header gets defaulted in parser + final parsed = Curl.parse(curl.toCurlString()); + expect(parsed.form, isTrue); + expect( + parsed.headers?[kHeaderContentType] ?? + parsed.headers?['content-type'], + 'multipart/form-data'); + }); }, ); + group('Roundtrip with body', () { + test('toCurlString includes body when present', () { + final curl = Curl( + method: 'POST', + uri: Uri.parse('https://api.apidash.dev/submit'), + data: 'a=1&b=2', + ); + final s = curl.toCurlString(); + final back = Curl.parse(s); + expect(back.method, 'POST'); + expect(back.data, 'a=1&b=2'); + expect(back.uri, Uri.parse('https://api.apidash.dev/submit')); + }); + }); + group( 'Special Parameters', () { diff --git a/packages/curl_parser/test/utility_test.dart b/packages/curl_parser/test/utility_test.dart index dc4ae96e..7329f7dd 100644 --- a/packages/curl_parser/test/utility_test.dart +++ b/packages/curl_parser/test/utility_test.dart @@ -88,4 +88,17 @@ void main() { ], ); }, timeout: defaultTimeout); + + test('split handles CRLF and backslash-newline', () async { + final args = splitAsCommandLineArgs( + "--request GET \\\r\n --url 'https://api.apidash.dev/echo' \\\n+ \n --header 'A: 1' "); + expect(args, [ + '--request', + 'GET', + '--url', + 'https://api.apidash.dev/echo', + '--header', + 'A: 1' + ]); + }, timeout: defaultTimeout); }