nodejs/firecrawl
1
0
Fork 0
mirror of https://github.com/firecrawl/firecrawl.git synced 2026-03-13 08:21:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(change-tracking): use diff library for standard unified diff format

Browse Source 
The git-diff npm package produces a simplified diff format (lines
prefixed with +/-/space, no chunk headers), which is incompatible with
parse-diff. This caused the json.files field in the change tracking
diff response to always be empty.

Replace git-diff with the diff library's createTwoFilesPatch, which
produces standard unified diff format with proper @@, ---, and +++
headers that parse-diff can parse correctly.

Co-Authored-By: micahstairs <micah@sideguide.dev>
This commit is contained in:
firecrawl-spring[bot]
2026-03-10 15:20:18 +00:00
parent 57a23fb3c5
commit 85de0dae4e
3 changed files with 11 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/package.json
View File

@@ -105,13 +105,13 @@
"cheerio": "^1.0.0-rc.12",
"cors": "^2.8.5",
"culori": "^4.0.2",
"diff": "^8.0.3",
"dotenv": "^16.3.1",
"esbuild": "^0.27.2",
"escape-html": "^1.0.3",
"express": "4.22.0",
"express-ws": "^5.0.2",
"geoip-country": "^5.0.202510312342",
"git-diff": "^2.0.6",
"http-cookie-agent": "^7.0.1",
"ioredis": "^5.6.1",
"ipaddr.js": "^2.2.0",

121
apps/api/pnpm-lock.yaml generated
View File

@@ -120,6 +120,9 @@ importers:
culori:
specifier: ^4.0.2
version: 4.0.2
diff:
specifier: ^8.0.3
version: 8.0.3
dotenv:
specifier: ^16.3.1
version: 16.4.5
@@ -138,9 +141,6 @@ importers:
geoip-country:
specifier: ^5.0.202510312342
version: 5.0.202510312342
git-diff:
specifier: ^2.0.6
version: 2.0.6
http-cookie-agent:
specifier: ^7.0.1
version: 7.0.1(tough-cookie@4.1.4)(undici@7.18.2)
@@ -2963,10 +2963,6 @@ packages:
resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==}
engines: {node: '>=12'}
ansi-styles@3.2.1:
resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}
engines: {node: '>=4'}
ansi-styles@4.3.0:
resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
engines: {node: '>=8'}
@@ -3166,10 +3162,6 @@ packages:
caniuse-lite@1.0.30001756:
resolution: {integrity: sha512-4HnCNKbMLkLdhJz3TToeVWHSnfJvPaq6vu/eRP0Ahub/07n484XHhBF5AJoSGHdVrS8tKFauUQz8Bp9P7LVx7A==}
chalk@2.4.2:
resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}
engines: {node: '>=4'}
chalk@4.1.2:
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
engines: {node: '>=10'}
@@ -3561,10 +3553,6 @@ packages:
escape-html@1.0.3:
resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==}
escape-string-regexp@1.0.5:
resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==}
engines: {node: '>=0.8.0'}
escape-string-regexp@2.0.0:
resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==}
engines: {node: '>=8'}
@@ -3816,10 +3804,6 @@ packages:
get-tsconfig@4.10.1:
resolution: {integrity: sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==}
git-diff@2.0.6:
resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}
engines: {node: '>= 4.8.0'}
glob-parent@5.1.2:
resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
engines: {node: '>= 6'}
@@ -3868,10 +3852,6 @@ packages:
engines: {node: '>=0.4.7'}
hasBin: true
has-flag@3.0.0:
resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}
engines: {node: '>=4'}
has-flag@4.0.0:
resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
engines: {node: '>=8'}
@@ -3980,10 +3960,6 @@ packages:
ini@1.3.8:
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
interpret@1.4.0:
resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==}
engines: {node: '>= 0.10'}
ioredis@5.6.1:
resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}
engines: {node: '>=12.22.0'}
@@ -4009,9 +3985,6 @@ packages:
is-buffer@1.1.6:
resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==}
is-core-module@2.13.1:
resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}
is-extglob@2.1.1:
resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
engines: {node: '>=0.10.0'}
@@ -4386,10 +4359,6 @@ packages:
resolution: {integrity: sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==}
engines: {node: '>= 12.0.0'}
loglevel@1.9.1:
resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==}
engines: {node: '>= 0.6.0'}
loose-envify@1.4.0:
resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}
hasBin: true
@@ -4748,9 +4717,6 @@ packages:
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
engines: {node: '>=8'}
path-parse@1.0.7:
resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
path-scurry@2.0.1:
resolution: {integrity: sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==}
engines: {node: 20 || >=22}
@@ -4942,10 +4908,6 @@ packages:
resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
rechoir@0.6.2:
resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==}
engines: {node: '>= 0.10'}
redis-errors@1.2.0:
resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
engines: {node: '>=4'}
@@ -4991,10 +4953,6 @@ packages:
resolve-pkg-maps@1.0.0:
resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==}
resolve@1.22.8:
resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}
hasBin: true
response-time@2.3.4:
resolution: {integrity: sha512-fiyq1RvW5/Br6iAtT8jN1XrNY8WPu2+yEypLbaijWry8WDZmn12azG9p/+c+qpEebURLlQmqCB8BNSu7ji+xQQ==}
engines: {node: '>= 0.8.0'}
@@ -5098,15 +5056,6 @@ packages:
resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==}
engines: {node: '>= 0.4'}
shelljs.exec@1.1.8:
resolution: {integrity: sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==}
engines: {node: '>= 4.0.0'}
shelljs@0.8.5:
resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==}
engines: {node: '>=4'}
hasBin: true
side-channel-list@1.0.0:
resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==}
engines: {node: '>= 0.4'}
@@ -5267,10 +5216,6 @@ packages:
engines: {node: '>=6.4.0'}
deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net
supports-color@5.5.0:
resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==}
engines: {node: '>=4'}
supports-color@7.2.0:
resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
engines: {node: '>=8'}
@@ -5279,10 +5224,6 @@ packages:
resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==}
engines: {node: '>=10'}
supports-preserve-symlinks-flag@1.0.0:
resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
engines: {node: '>= 0.4'}
symbol-tree@3.2.4:
resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==}
@@ -8490,10 +8431,6 @@ snapshots:
ansi-regex@6.0.1: {}
ansi-styles@3.2.1:
dependencies:
color-convert: 1.9.3
ansi-styles@4.3.0:
dependencies:
color-convert: 2.0.1
@@ -8728,12 +8665,6 @@ snapshots:
caniuse-lite@1.0.30001756: {}
chalk@2.4.2:
dependencies:
ansi-styles: 3.2.1
escape-string-regexp: 1.0.5
supports-color: 5.5.0
chalk@4.1.2:
dependencies:
ansi-styles: 4.3.0
@@ -9117,8 +9048,6 @@ snapshots:
escape-html@1.0.3: {}
escape-string-regexp@1.0.5: {}
escape-string-regexp@2.0.0: {}
etag@1.8.1: {}
@@ -9447,14 +9376,6 @@ snapshots:
dependencies:
resolve-pkg-maps: 1.0.0
git-diff@2.0.6:
dependencies:
chalk: 2.4.2
diff: 8.0.3
loglevel: 1.9.1
shelljs: 0.8.5
shelljs.exec: 1.1.8
glob-parent@5.1.2:
dependencies:
is-glob: 4.0.3
@@ -9530,8 +9451,6 @@ snapshots:
optionalDependencies:
uglify-js: 3.19.3
has-flag@3.0.0: {}
has-flag@4.0.0: {}
has-symbols@1.1.0: {}
@@ -9652,8 +9571,6 @@ snapshots:
ini@1.3.8: {}
interpret@1.4.0: {}
ioredis@5.6.1:
dependencies:
'@ioredis/commands': 1.2.0
@@ -9688,10 +9605,6 @@ snapshots:
is-buffer@1.1.6: {}
is-core-module@2.13.1:
dependencies:
hasown: 2.0.2
is-extglob@2.1.1: {}
is-fullwidth-code-point@3.0.0: {}
@@ -10324,8 +10237,6 @@ snapshots:
safe-stable-stringify: 2.4.3
triple-beam: 1.4.1
loglevel@1.9.1: {}
loose-envify@1.4.0:
dependencies:
js-tokens: 4.0.0
@@ -10643,8 +10554,6 @@ snapshots:
path-key@3.1.1: {}
path-parse@1.0.7: {}
path-scurry@2.0.1:
dependencies:
lru-cache: 11.2.5
@@ -10825,10 +10734,6 @@ snapshots:
process: 0.11.10
string_decoder: 1.3.0
rechoir@0.6.2:
dependencies:
resolve: 1.22.8
redis-errors@1.2.0: {}
redis-info@3.1.0:
@@ -10871,12 +10776,6 @@ snapshots:
resolve-pkg-maps@1.0.0: {}
resolve@1.22.8:
dependencies:
is-core-module: 2.13.1
path-parse: 1.0.7
supports-preserve-symlinks-flag: 1.0.0
response-time@2.3.4:
dependencies:
depd: 2.0.0
@@ -10992,14 +10891,6 @@ snapshots:
shell-quote@1.8.3: {}
shelljs.exec@1.1.8: {}
shelljs@0.8.5:
dependencies:
glob: 7.2.3
interpret: 1.4.0
rechoir: 0.6.2
side-channel-list@1.0.0:
dependencies:
es-errors: 1.3.0
@@ -11171,10 +11062,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
supports-color@5.5.0:
dependencies:
has-flag: 3.0.0
supports-color@7.2.0:
dependencies:
has-flag: 4.0.0
@@ -11183,8 +11070,6 @@ snapshots:
dependencies:
has-flag: 4.0.0
supports-preserve-symlinks-flag@1.0.0: {}
symbol-tree@3.2.4: {}
synckit@0.11.11:

12
apps/api/src/scraper/scrapeURL/transformers/diff.ts
View File

@@ -1,7 +1,7 @@
import { supabase_service } from "../../../services/supabase";
import { Document } from "../../../controllers/v1/types";
import { Meta } from "../index";
import gitDiff from "git-diff";
import { createTwoFilesPatch } from "diff";
import parseDiff from "parse-diff";
import { generateCompletions } from "./llmExtract";
import { hasFormatOfType } from "../../../lib/format-utils";
@@ -152,10 +152,12 @@ export async function deriveDiff(
changeTrackingFormat?.modes?.includes("git-diff") &&
changeStatus === "changed"
) {
const diffText = gitDiff(previousMarkdown, currentMarkdown, {
color: false,
wordDiff: false,
});
const diffText = createTwoFilesPatch(
"previous",
"current",
previousMarkdown,
currentMarkdown,
);
// meta.logger.debug("Diff text", { diffText });
if (diffText) {
const diffStructured = parseDiff(diffText);