diff --git a/apps/api/package.json b/apps/api/package.json index bd162867e..f6ac2a524 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -105,13 +105,13 @@ "cheerio": "^1.0.0-rc.12", "cors": "^2.8.5", "culori": "^4.0.2", + "diff": "^8.0.3", "dotenv": "^16.3.1", "esbuild": "^0.27.2", "escape-html": "^1.0.3", "express": "4.22.0", "express-ws": "^5.0.2", "geoip-country": "^5.0.202510312342", - "git-diff": "^2.0.6", "http-cookie-agent": "^7.0.1", "ioredis": "^5.6.1", "ipaddr.js": "^2.2.0", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 3e55f106c..16eb822ef 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -120,6 +120,9 @@ importers: culori: specifier: ^4.0.2 version: 4.0.2 + diff: + specifier: ^8.0.3 + version: 8.0.3 dotenv: specifier: ^16.3.1 version: 16.4.5 @@ -138,9 +141,6 @@ importers: geoip-country: specifier: ^5.0.202510312342 version: 5.0.202510312342 - git-diff: - specifier: ^2.0.6 - version: 2.0.6 http-cookie-agent: specifier: ^7.0.1 version: 7.0.1(tough-cookie@4.1.4)(undici@7.18.2) @@ -2963,10 +2963,6 @@ packages: resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==} engines: {node: '>=12'} - ansi-styles@3.2.1: - resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} - engines: {node: '>=4'} - ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} @@ -3166,10 +3162,6 @@ packages: caniuse-lite@1.0.30001756: resolution: {integrity: sha512-4HnCNKbMLkLdhJz3TToeVWHSnfJvPaq6vu/eRP0Ahub/07n484XHhBF5AJoSGHdVrS8tKFauUQz8Bp9P7LVx7A==} - chalk@2.4.2: - resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} - engines: {node: '>=4'} - chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -3561,10 +3553,6 @@ packages: escape-html@1.0.3: resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} - escape-string-regexp@1.0.5: - resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} - engines: {node: '>=0.8.0'} - escape-string-regexp@2.0.0: resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} engines: {node: '>=8'} @@ -3816,10 +3804,6 @@ packages: get-tsconfig@4.10.1: resolution: {integrity: sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==} - git-diff@2.0.6: - resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==} - engines: {node: '>= 4.8.0'} - glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -3868,10 +3852,6 @@ packages: engines: {node: '>=0.4.7'} hasBin: true - has-flag@3.0.0: - resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} - engines: {node: '>=4'} - has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -3980,10 +3960,6 @@ packages: ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - interpret@1.4.0: - resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==} - engines: {node: '>= 0.10'} - ioredis@5.6.1: resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==} engines: {node: '>=12.22.0'} @@ -4009,9 +3985,6 @@ packages: is-buffer@1.1.6: resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} - is-core-module@2.13.1: - resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==} - is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -4386,10 +4359,6 @@ packages: resolution: {integrity: sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==} engines: {node: '>= 12.0.0'} - loglevel@1.9.1: - resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==} - engines: {node: '>= 0.6.0'} - loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -4748,9 +4717,6 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} - path-parse@1.0.7: - resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} - path-scurry@2.0.1: resolution: {integrity: sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==} engines: {node: 20 || >=22} @@ -4942,10 +4908,6 @@ packages: resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - rechoir@0.6.2: - resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} - engines: {node: '>= 0.10'} - redis-errors@1.2.0: resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} engines: {node: '>=4'} @@ -4991,10 +4953,6 @@ packages: resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - resolve@1.22.8: - resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} - hasBin: true - response-time@2.3.4: resolution: {integrity: sha512-fiyq1RvW5/Br6iAtT8jN1XrNY8WPu2+yEypLbaijWry8WDZmn12azG9p/+c+qpEebURLlQmqCB8BNSu7ji+xQQ==} engines: {node: '>= 0.8.0'} @@ -5098,15 +5056,6 @@ packages: resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} engines: {node: '>= 0.4'} - shelljs.exec@1.1.8: - resolution: {integrity: sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==} - engines: {node: '>= 4.0.0'} - - shelljs@0.8.5: - resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} - engines: {node: '>=4'} - hasBin: true - side-channel-list@1.0.0: resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} engines: {node: '>= 0.4'} @@ -5267,10 +5216,6 @@ packages: engines: {node: '>=6.4.0'} deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net - supports-color@5.5.0: - resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} - engines: {node: '>=4'} - supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -5279,10 +5224,6 @@ packages: resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} engines: {node: '>=10'} - supports-preserve-symlinks-flag@1.0.0: - resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} - engines: {node: '>= 0.4'} - symbol-tree@3.2.4: resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} @@ -8490,10 +8431,6 @@ snapshots: ansi-regex@6.0.1: {} - ansi-styles@3.2.1: - dependencies: - color-convert: 1.9.3 - ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 @@ -8728,12 +8665,6 @@ snapshots: caniuse-lite@1.0.30001756: {} - chalk@2.4.2: - dependencies: - ansi-styles: 3.2.1 - escape-string-regexp: 1.0.5 - supports-color: 5.5.0 - chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -9117,8 +9048,6 @@ snapshots: escape-html@1.0.3: {} - escape-string-regexp@1.0.5: {} - escape-string-regexp@2.0.0: {} etag@1.8.1: {} @@ -9447,14 +9376,6 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 - git-diff@2.0.6: - dependencies: - chalk: 2.4.2 - diff: 8.0.3 - loglevel: 1.9.1 - shelljs: 0.8.5 - shelljs.exec: 1.1.8 - glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -9530,8 +9451,6 @@ snapshots: optionalDependencies: uglify-js: 3.19.3 - has-flag@3.0.0: {} - has-flag@4.0.0: {} has-symbols@1.1.0: {} @@ -9652,8 +9571,6 @@ snapshots: ini@1.3.8: {} - interpret@1.4.0: {} - ioredis@5.6.1: dependencies: '@ioredis/commands': 1.2.0 @@ -9688,10 +9605,6 @@ snapshots: is-buffer@1.1.6: {} - is-core-module@2.13.1: - dependencies: - hasown: 2.0.2 - is-extglob@2.1.1: {} is-fullwidth-code-point@3.0.0: {} @@ -10324,8 +10237,6 @@ snapshots: safe-stable-stringify: 2.4.3 triple-beam: 1.4.1 - loglevel@1.9.1: {} - loose-envify@1.4.0: dependencies: js-tokens: 4.0.0 @@ -10643,8 +10554,6 @@ snapshots: path-key@3.1.1: {} - path-parse@1.0.7: {} - path-scurry@2.0.1: dependencies: lru-cache: 11.2.5 @@ -10825,10 +10734,6 @@ snapshots: process: 0.11.10 string_decoder: 1.3.0 - rechoir@0.6.2: - dependencies: - resolve: 1.22.8 - redis-errors@1.2.0: {} redis-info@3.1.0: @@ -10871,12 +10776,6 @@ snapshots: resolve-pkg-maps@1.0.0: {} - resolve@1.22.8: - dependencies: - is-core-module: 2.13.1 - path-parse: 1.0.7 - supports-preserve-symlinks-flag: 1.0.0 - response-time@2.3.4: dependencies: depd: 2.0.0 @@ -10992,14 +10891,6 @@ snapshots: shell-quote@1.8.3: {} - shelljs.exec@1.1.8: {} - - shelljs@0.8.5: - dependencies: - glob: 7.2.3 - interpret: 1.4.0 - rechoir: 0.6.2 - side-channel-list@1.0.0: dependencies: es-errors: 1.3.0 @@ -11171,10 +11062,6 @@ snapshots: transitivePeerDependencies: - supports-color - supports-color@5.5.0: - dependencies: - has-flag: 3.0.0 - supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -11183,8 +11070,6 @@ snapshots: dependencies: has-flag: 4.0.0 - supports-preserve-symlinks-flag@1.0.0: {} - symbol-tree@3.2.4: {} synckit@0.11.11: diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.ts index 16850f846..246d00466 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts @@ -1,7 +1,7 @@ import { supabase_service } from "../../../services/supabase"; import { Document } from "../../../controllers/v1/types"; import { Meta } from "../index"; -import gitDiff from "git-diff"; +import { createTwoFilesPatch } from "diff"; import parseDiff from "parse-diff"; import { generateCompletions } from "./llmExtract"; import { hasFormatOfType } from "../../../lib/format-utils"; @@ -152,10 +152,12 @@ export async function deriveDiff( changeTrackingFormat?.modes?.includes("git-diff") && changeStatus === "changed" ) { - const diffText = gitDiff(previousMarkdown, currentMarkdown, { - color: false, - wordDiff: false, - }); + const diffText = createTwoFilesPatch( + "previous", + "current", + previousMarkdown, + currentMarkdown, + ); // meta.logger.debug("Diff text", { diffText }); if (diffText) { const diffStructured = parseDiff(diffText);