From b53cac66de4032512b067d2cc313243b76117931 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Mon, 5 Oct 2026 19:13:33 +0900 Subject: [PATCH 1/2] Store the language of remote posts with contentMap Mastodon and Hollo send content together with contentMap, so after JSON-LD expansion the content property holds both a plain string and a language-tagged one. persistPost() only looked at the singular object.content and object.summary accessors, which return the plain string, so the language of such posts was always stored as null. The language is now taken from the first LanguageString found among all content values, then all summary values. Values tagged as undetermined (und), which a context-level default @language such as Pleroma's produces, are skipped so that they don't shadow the real language in contentMap. Fixes https://github.com/fedify-dev/hollo/issues/647 Assisted-by: Claude Code:claude-opus-5-5 Assisted-by: Claude Code:claude-fable-5-1 --- CHANGES.md | 6 ++++ src/federation/post.test.ts | 71 +++++++++++++++++++++++++++++++++++++ src/federation/post.ts | 22 ++++++++---- 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 36f98599..7d633a32 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,12 @@ Version 0.9.22 To be released. + - Fixed the language of remote posts not being stored when they were + received from Mastodon, Hollo, or any other software that sends `content` + together with `contentMap`. [[#647]] + +[#647]: https://github.com/fedify-dev/hollo/issues/647 + Version 0.9.21 -------------- diff --git a/src/federation/post.test.ts b/src/federation/post.test.ts index afd5659f..43417dfb 100644 --- a/src/federation/post.test.ts +++ b/src/federation/post.test.ts @@ -755,6 +755,77 @@ describe("persistPost", () => { { accountId: mentioned.id, postId: result.id }, ]); }); + + it("stores the language when content is accompanied by contentMap", async () => { + expect.assertions(2); + const author = await seedRemoteAccount("author"); + const object = await Note.fromJsonLd({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Note", + id: "https://remote.test/@author/posts/content-map", + content: "

こんにちは

", + contentMap: { ja: "

こんにちは

" }, + to: PUBLIC_COLLECTION.href, + }); + + const result = await persistPost( + db, + object.clone({ attribution: createPerson(author) }), + "https://hollo.test", + { account: author }, + ); + + expect(result?.contentHtml).toBe("

こんにちは

"); + expect(result?.language).toBe("ja"); + }); + + it("stores the language from summaryMap when content has none", async () => { + expect.assertions(1); + const author = await seedRemoteAccount("author"); + const object = await Note.fromJsonLd({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Note", + id: "https://remote.test/@author/posts/summary-map", + summary: "CW", + summaryMap: { "zh-TW": "CW" }, + content: "

內容

", + to: PUBLIC_COLLECTION.href, + }); + + const result = await persistPost( + db, + object.clone({ attribution: createPerson(author) }), + "https://hollo.test", + { account: author }, + ); + + expect(result?.language).toBe("zh-TW"); + }); + + it("ignores undetermined languages from a default @language", async () => { + expect.assertions(1); + const author = await seedRemoteAccount("author"); + const object = await Note.fromJsonLd({ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { "@language": "und" }, + ], + type: "Note", + id: "https://remote.test/@author/posts/default-language", + content: "

こんにちは

", + contentMap: { ja: "

こんにちは

" }, + to: PUBLIC_COLLECTION.href, + }); + + const result = await persistPost( + db, + object.clone({ attribution: createPerson(author) }), + "https://hollo.test", + { account: author }, + ); + + expect(result?.language).toBe("ja"); + }); }); describe("toObject", () => { diff --git a/src/federation/post.ts b/src/federation/post.ts index 401a54dc..97dc02ee 100644 --- a/src/federation/post.ts +++ b/src/federation/post.ts @@ -119,6 +119,21 @@ function getQuoteApprovalPolicy( return "nobody"; } +// Mastodon and Hollo send both `content` and `contentMap`, so after JSON-LD +// expansion the singular `content` accessor returns the plain string rather +// than the LanguageString; search all values instead. Values tagged `und` +// (e.g., from a context-level default `@language`) are skipped, and recent +// engines report `undefined` instead of `"und"` as their language: +function getLanguage(object: ASPost): string | null { + for (const value of [...object.contents, ...object.summaries]) { + if (!(value instanceof LanguageString)) continue; + const language = value.locale.language; + if (language == null || language === "und") continue; + return value.locale.toString(); + } + return null; +} + async function getVerifiedQuoteAuthorizationIri( object: ASPost, quoteTargetIri: string | null, @@ -378,12 +393,7 @@ export async function persistPost( : "direct", summary: object.summary?.toString(), contentHtml: object.content?.toString(), - language: - object.content instanceof LanguageString - ? object.content.locale.toString() - : object.summary instanceof LanguageString - ? object.summary.locale.toString() - : null, + language: getLanguage(object), previewCard, tags, emojis, From 2160706411fa82b6fb23e272d9b675b27d56ffef Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Mon, 5 Oct 2026 19:15:50 +0900 Subject: [PATCH 2/2] Upgrade Fedify to 2.2.16 Fedify 2.2.16 normalizes BCP 47 extended language subtags before handing language tags to Intl.Locale. Mastodon tags Cantonese posts as zh-YUE, which is a valid BCP 47 tag but not a valid Unicode locale identifier, so Fedify used to drop such language-tagged strings and Hollo stored no language for those posts. They are now kept with the canonical tag yue. https://github.com/fedify-dev/hollo/issues/647 https://github.com/fedify-dev/fedify/issues/1229 Assisted-by: Claude Code:claude-opus-5-5 --- CHANGES.md | 10 +++- package.json | 14 ++--- pnpm-lock.yaml | 112 ++++++++++++++++++------------------ src/federation/post.test.ts | 22 +++++++ 4 files changed, 94 insertions(+), 64 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7d633a32..68211168 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,9 +8,17 @@ To be released. - Fixed the language of remote posts not being stored when they were received from Mastodon, Hollo, or any other software that sends `content` - together with `contentMap`. [[#647]] + together with `contentMap`. [[#647], [#659]] + - Upgraded Fedify to 2.2.16, which normalizes BCP 47 extended language + subtags before parsing language tags. Language-tagged strings with tags + like `zh-YUE`, which Mastodon uses for Cantonese, were previously dropped + because `Intl.Locale` rejects them; they are now kept with the canonical + tag (e.g., `yue`). [[#647], [fedify-dev/fedify#1229], [#659]] + +[fedify-dev/fedify#1229]: https://github.com/fedify-dev/fedify/issues/1229 [#647]: https://github.com/fedify-dev/hollo/issues/647 +[#659]: https://github.com/fedify-dev/hollo/pull/659 Version 0.9.21 diff --git a/package.json b/package.json index 8cb41d88..ade20020 100644 --- a/package.json +++ b/package.json @@ -32,14 +32,14 @@ }, "dependencies": { "@aws-sdk/credential-providers": "^3.1037.0", - "@fedify/debugger": "~2.2.15", - "@fedify/fedify": "~2.2.15", - "@fedify/hono": "~2.2.15", + "@fedify/debugger": "~2.2.16", + "@fedify/fedify": "~2.2.16", + "@fedify/hono": "~2.2.16", "@fedify/markdown-it-hashtag": "~0.3.0", "@fedify/markdown-it-mention": "~0.3.0", - "@fedify/postgres": "~2.2.15", - "@fedify/vocab": "~2.2.15", - "@fedify/webfinger": "~2.2.15", + "@fedify/postgres": "~2.2.16", + "@fedify/vocab": "~2.2.16", + "@fedify/webfinger": "~2.2.16", "@hexagon/base64": "^2.0.4", "@hono/node-server": "^1.19.13", "@hono/zod-validator": "^0.7.6", @@ -88,7 +88,7 @@ }, "devDependencies": { "@dotenvx/dotenvx": "^1.52.0", - "@fedify/lint": "~2.2.15", + "@fedify/lint": "~2.2.16", "@iconify-json/lucide": "^1.2.104", "@reporters/github": "^1.13.1", "@types/fluent-ffmpeg": "^2.1.28", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2388cd22..e8b3c370 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,14 +16,14 @@ importers: specifier: ^3.1037.0 version: 3.1037.0 '@fedify/debugger': - specifier: ~2.2.15 - version: 2.2.15(@fedify/fedify@2.2.15) + specifier: ~2.2.16 + version: 2.2.16(@fedify/fedify@2.2.16) '@fedify/fedify': - specifier: ~2.2.15 - version: 2.2.15 + specifier: ~2.2.16 + version: 2.2.16 '@fedify/hono': - specifier: ~2.2.15 - version: 2.2.15(@fedify/fedify@2.2.15)(hono@4.12.21) + specifier: ~2.2.16 + version: 2.2.16(@fedify/fedify@2.2.16)(hono@4.12.21) '@fedify/markdown-it-hashtag': specifier: ~0.3.0 version: 0.3.0 @@ -31,14 +31,14 @@ importers: specifier: ~0.3.0 version: 0.3.0 '@fedify/postgres': - specifier: ~2.2.15 - version: 2.2.15(@fedify/fedify@2.2.15)(postgres@3.4.9) + specifier: ~2.2.16 + version: 2.2.16(@fedify/fedify@2.2.16)(postgres@3.4.9) '@fedify/vocab': - specifier: ~2.2.15 - version: 2.2.15 + specifier: ~2.2.16 + version: 2.2.16 '@fedify/webfinger': - specifier: ~2.2.15 - version: 2.2.15 + specifier: ~2.2.16 + version: 2.2.16 '@hexagon/base64': specifier: ^2.0.4 version: 2.0.4 @@ -179,8 +179,8 @@ importers: specifier: ^1.52.0 version: 1.52.0 '@fedify/lint': - specifier: ~2.2.15 - version: 2.2.15(@fedify/fedify@2.2.15)(eslint@10.2.1(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@6.0.3) + specifier: ~2.2.16 + version: 2.2.16(@fedify/fedify@2.2.16)(eslint@10.2.1(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@6.0.3) '@iconify-json/lucide': specifier: ^1.2.104 version: 1.2.108 @@ -1285,25 +1285,25 @@ packages: peerDependencies: '@opentelemetry/api': ^1.9.0 - '@fedify/debugger@2.2.15': - resolution: {integrity: sha512-fXxVHt4XZiYSnL+yQACeeUTxaWjOUVDgLczsK7h/YkVb3tmH7d2tsv7pD2xFqzoYmj6tn2gwPkvb/iBFMkfN4A==} + '@fedify/debugger@2.2.16': + resolution: {integrity: sha512-I5YB6XVymmjvuMjqifUYNqNWDRbDikZYzBEvjt10gaDupHDI6UdaheRMz7GzP1PHt61xj4kaD6zULfU9EuPFdw==} peerDependencies: - '@fedify/fedify': ^2.2.15 + '@fedify/fedify': ^2.2.16 - '@fedify/fedify@2.2.15': - resolution: {integrity: sha512-+lic+a3kxzRsTbGKmX+1l8Mq4Va8kawSwDN8LplYdAgEK/bNeOqFOpbUob9pry1xy3PikegSGkBomwN0xIw+3w==} + '@fedify/fedify@2.2.16': + resolution: {integrity: sha512-WPOr1nOFp2RkDo98lqJ42gqCadN3+NbA7qe4fWcz5cNQ1SUtNnt8XZsGyre2vVANP4E246XJhtVLf9J0Jbvtgg==} engines: {bun: '>=1.1.0', deno: '>=2.0.0', node: '>=22.0.0'} - '@fedify/hono@2.2.15': - resolution: {integrity: sha512-JmwFWj3+Q8sxCINssrorDMEUogI2uk+zpY44IJgYwaERRzuGtrOsWoCL6guAbUCHMUJphWKwaLTXzM74v304eg==} + '@fedify/hono@2.2.16': + resolution: {integrity: sha512-0/fR8pCI2BAVNEFWEM8LdcMdyBjXVPP/6/6Xozg3hxNin+mQYDUaRBRyl2ACVCVNvddGC39zuWUokzUPlLTF3g==} peerDependencies: - '@fedify/fedify': ^2.2.15 + '@fedify/fedify': ^2.2.16 hono: ^4.0.0 - '@fedify/lint@2.2.15': - resolution: {integrity: sha512-EdzscJId35J01c4x5Cn7tHgIpMlptyTcfkUyvzS9LYfUZTTZF2sL8hAjUX1R7noAUucFja2EcVjy15rMtB7Kkg==} + '@fedify/lint@2.2.16': + resolution: {integrity: sha512-i7hO14wfaYfpWl2NoXUDIPaGf/hH+hwRsmbmA4W1B6CHmbJu7FXhpucKcwVf7D8yFRnmZW9ZN8G8vHdz63291w==} peerDependencies: - '@fedify/fedify': ^2.2.15 + '@fedify/fedify': ^2.2.16 eslint: '>=9.0.0' peerDependenciesMeta: eslint: @@ -1315,26 +1315,26 @@ packages: '@fedify/markdown-it-mention@0.3.0': resolution: {integrity: sha512-4uVELsxh9AW0hzC6BVFqN59YWj6M/p6AZpwb/CQDjvcinfHyQp2yYtGDV4fBgsE5WC/ukpvU1pBNH20ts8TK4w==} - '@fedify/postgres@2.2.15': - resolution: {integrity: sha512-bn7fPiD9AxA4Hasy/mCSCT43kJ6P8/VyBmpOZ2NwPvwleZNyQj0ZNpxwcye1IbgSsKXslFIrDGgiTZGEWFOqAQ==} + '@fedify/postgres@2.2.16': + resolution: {integrity: sha512-UZQoK0ajqxVsLPhCuIk86aLNb6VFoctsN0qifR45mEkixczy8vr3xe5EN82xnrmsjL+H3ORVwwtFfZSA1m1m1w==} peerDependencies: - '@fedify/fedify': ^2.2.15 + '@fedify/fedify': ^2.2.16 postgres: ^3.4.7 - '@fedify/vocab-runtime@2.2.15': - resolution: {integrity: sha512-4yc01oWu4NNevXPND0y4+oSYhBr5AnQhNHxR2szDsZOLHfl59byAJPG61ntOFqD8d/bO6iKASPZhDO6DQI2qYw==} + '@fedify/vocab-runtime@2.2.16': + resolution: {integrity: sha512-/V4UmGPffBbEo6DYECnu1+s3glaSm5Yfct/AjOhKhh7YO59Qw6K1ZxwUs3PnROjNJh6c/rV+4WgcyVFMQLnsug==} engines: {bun: '>=1.1.0', deno: '>=2.0.0', node: '>=22.0.0'} - '@fedify/vocab-tools@2.2.15': - resolution: {integrity: sha512-dPNJgYGorg1P2cyGGPrf77YFj3ftUnrhEQVYn0u6cIMu3qQcMwJ0QEXKgkVxM6fWZC1ULvxdkOrL+myrO0AIhQ==} + '@fedify/vocab-tools@2.2.16': + resolution: {integrity: sha512-9bZZLV7FNCe43nue/mehG3cUpckSXdCYXrwJHhBkSsbKwfTnyeOWXAncXa3Hvh1feDMNd+0eckKWtRps5irGrQ==} engines: {bun: '>=1.1.0', deno: '>=2.0.0', node: '>=22.0.0'} - '@fedify/vocab@2.2.15': - resolution: {integrity: sha512-d59YCmNfKo/NpH6dOo/W3wEjMWPDBfIEDHxRuflHjtS7OcmH20oiju3uHNgrhj5yE2uOLlaOqPRCtUm2++gIdg==} + '@fedify/vocab@2.2.16': + resolution: {integrity: sha512-IFZENI1FaD/zdcGK9atfdsFI7uwWTXcKLgRDsOS80zcmTyorgzXiBM8hTjSeF+TAew+df1HndKtA1zm7sJnIJw==} engines: {bun: '>=1.1.0', deno: '>=2.0.0', node: '>=22.0.0'} - '@fedify/webfinger@2.2.15': - resolution: {integrity: sha512-REW0yp+CCdtI7WdqBRYN3gFgr/HdiN2TD6RL+VXmUCSlC+qxSR0wCvFuZ5T/nk788vXnrcQE0+C3q0F0ZfBGuA==} + '@fedify/webfinger@2.2.16': + resolution: {integrity: sha512-7HFgpTULheYgUyUsCbtU821DVYsoN5vJzjz15NZL86w6J3bCr7tgmXTjhcKD4Jen7KkG7Z0BgjD5t+AC73340Q==} engines: {bun: '>=1.1.0', deno: '>=2.0.0', node: '>=22.0.0'} '@fxts/core@1.26.0': @@ -8126,9 +8126,9 @@ snapshots: transitivePeerDependencies: - supports-color - '@fedify/debugger@2.2.15(@fedify/fedify@2.2.15)': + '@fedify/debugger@2.2.16(@fedify/fedify@2.2.16)': dependencies: - '@fedify/fedify': 2.2.15 + '@fedify/fedify': 2.2.16 '@js-temporal/polyfill': 0.5.1 '@logtape/logtape': 2.1.1 '@opentelemetry/api': 1.9.1 @@ -8137,11 +8137,11 @@ snapshots: '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.1) hono: 4.12.21 - '@fedify/fedify@2.2.15': + '@fedify/fedify@2.2.16': dependencies: - '@fedify/vocab': 2.2.15 - '@fedify/vocab-runtime': 2.2.15 - '@fedify/webfinger': 2.2.15 + '@fedify/vocab': 2.2.16 + '@fedify/vocab-runtime': 2.2.16 + '@fedify/webfinger': 2.2.16 '@js-temporal/polyfill': 0.5.1 '@logtape/logtape': 2.1.1 '@opentelemetry/api': 1.9.1 @@ -8157,14 +8157,14 @@ snapshots: url-template: 3.1.1 urlpattern-polyfill: 10.1.0 - '@fedify/hono@2.2.15(@fedify/fedify@2.2.15)(hono@4.12.21)': + '@fedify/hono@2.2.16(@fedify/fedify@2.2.16)(hono@4.12.21)': dependencies: - '@fedify/fedify': 2.2.15 + '@fedify/fedify': 2.2.16 hono: 4.12.21 - '@fedify/lint@2.2.15(@fedify/fedify@2.2.15)(eslint@10.2.1(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@6.0.3)': + '@fedify/lint@2.2.16(@fedify/fedify@2.2.16)(eslint@10.2.1(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@6.0.3)': dependencies: - '@fedify/fedify': 2.2.15 + '@fedify/fedify': 2.2.16 '@fxts/core': 1.26.0 '@typescript-eslint/parser': 8.59.0(eslint@10.2.1(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@6.0.3) '@typescript-eslint/utils': 8.59.0(eslint@10.2.1(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@6.0.3) @@ -8182,14 +8182,14 @@ snapshots: dependencies: markdown-it: 14.1.1 - '@fedify/postgres@2.2.15(@fedify/fedify@2.2.15)(postgres@3.4.9)': + '@fedify/postgres@2.2.16(@fedify/fedify@2.2.16)(postgres@3.4.9)': dependencies: - '@fedify/fedify': 2.2.15 + '@fedify/fedify': 2.2.16 '@js-temporal/polyfill': 0.5.1 '@logtape/logtape': 2.1.1 postgres: 3.4.9 - '@fedify/vocab-runtime@2.2.15': + '@fedify/vocab-runtime@2.2.16': dependencies: '@js-temporal/polyfill': 0.5.1 '@logtape/logtape': 2.1.1 @@ -8200,18 +8200,18 @@ snapshots: jsonld: 9.0.0 pkijs: 3.4.0 - '@fedify/vocab-tools@2.2.15': + '@fedify/vocab-tools@2.2.16': dependencies: '@cfworker/json-schema': 4.1.1 byte-encodings: 1.0.11 es-toolkit: 1.46.1 yaml: 2.9.0 - '@fedify/vocab@2.2.15': + '@fedify/vocab@2.2.16': dependencies: - '@fedify/vocab-runtime': 2.2.15 - '@fedify/vocab-tools': 2.2.15 - '@fedify/webfinger': 2.2.15 + '@fedify/vocab-runtime': 2.2.16 + '@fedify/vocab-tools': 2.2.16 + '@fedify/webfinger': 2.2.16 '@js-temporal/polyfill': 0.5.1 '@logtape/logtape': 2.1.1 '@multiformats/base-x': 4.0.1 @@ -8221,9 +8221,9 @@ snapshots: jsonld: 9.0.0 pkijs: 3.4.0 - '@fedify/webfinger@2.2.15': + '@fedify/webfinger@2.2.16': dependencies: - '@fedify/vocab-runtime': 2.2.15 + '@fedify/vocab-runtime': 2.2.16 '@logtape/logtape': 2.1.1 '@opentelemetry/api': 1.9.1 es-toolkit: 1.43.0 diff --git a/src/federation/post.test.ts b/src/federation/post.test.ts index 43417dfb..99d3e445 100644 --- a/src/federation/post.test.ts +++ b/src/federation/post.test.ts @@ -802,6 +802,28 @@ describe("persistPost", () => { expect(result?.language).toBe("zh-TW"); }); + it("normalizes extended language tags such as zh-YUE", async () => { + expect.assertions(1); + const author = await seedRemoteAccount("author"); + const object = await Note.fromJsonLd({ + "@context": "https://www.w3.org/ns/activitystreams", + type: "Note", + id: "https://remote.test/@author/posts/extlang", + content: "

你好

", + contentMap: { "zh-YUE": "

你好

" }, + to: PUBLIC_COLLECTION.href, + }); + + const result = await persistPost( + db, + object.clone({ attribution: createPerson(author) }), + "https://hollo.test", + { account: author }, + ); + + expect(result?.language).toBe("yue"); + }); + it("ignores undetermined languages from a default @language", async () => { expect.assertions(1); const author = await seedRemoteAccount("author");