Merge pull request #1924 from famedly/nico/unlocalizedBody

Fix various edge cases in the unlocalized body calculation
This commit is contained in:
td 2024-10-16 14:01:59 +02:00 committed by GitHub
commit 3e35120aa1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 627 additions and 25 deletions

View File

@ -292,15 +292,18 @@ class Event extends MatrixEvent {
String get body {
if (redacted) return 'Redacted';
if (text != '') return text;
if (formattedText != '') return formattedText;
return type;
}
/// Use this to get a plain-text representation of the event, stripping things
/// like spoilers and thelike. Useful for plain text notifications.
String get plaintextBody => content['format'] == 'org.matrix.custom.html'
? HtmlToText.convert(formattedText)
: body;
String get plaintextBody => switch (formattedText) {
// if the formattedText is empty, fallback to body
'' => body,
final String s when content['format'] == 'org.matrix.custom.html' =>
HtmlToText.convert(s),
_ => body,
};
/// Returns a list of [Receipt] instances for this event.
List<Receipt> get receipts {
@ -762,7 +765,10 @@ class Event extends MatrixEvent {
/// Returns a localized String representation of this event. For a
/// room list you may find [withSenderNamePrefix] useful. Set [hideReply] to
/// crop all lines starting with '>'. With [plaintextBody] it'll use the
/// plaintextBody instead of the normal body.
/// plaintextBody instead of the normal body which in practice will convert
/// the html body to a plain text body before falling back to the body. In
/// either case this function won't return the html body without converting
/// it to plain text.
/// [removeMarkdown] allow to remove the markdown formating from the event body.
/// Usefull form message preview or notifications text.
Future<String> calcLocalizedBody(MatrixLocalizations i18n,
@ -851,37 +857,44 @@ class Event extends MatrixEvent {
}
/// Calculating the body of an event regardless of localization.
String calcUnlocalizedBody(
{bool hideReply = false,
String calcUnlocalizedBody({
bool hideReply = false,
bool hideEdit = false,
bool plaintextBody = false,
bool removeMarkdown = false}) {
bool removeMarkdown = false,
}) {
if (redacted) {
return 'Removed by ${senderFromMemoryOrFallback.displayName ?? senderId}';
}
var body = plaintextBody ? this.plaintextBody : this.body;
// we need to know if the message is an html message to be able to determine
// if we need to strip the reply fallback.
var htmlMessage = content['format'] != 'org.matrix.custom.html';
// Html messages will already have their reply fallback removed during the Html to Text conversion.
var mayHaveReplyFallback = !plaintextBody ||
(content['format'] != 'org.matrix.custom.html' ||
formattedText.isEmpty);
// If we have an edit, we want to operate on the new content
final newContent = content.tryGetMap<String, Object?>('m.new_content');
if (hideEdit &&
relationshipType == RelationshipTypes.edit &&
newContent != null) {
if (plaintextBody && newContent['format'] == 'org.matrix.custom.html') {
htmlMessage = true;
body = HtmlToText.convert(
newContent.tryGet<String>('formatted_body') ?? formattedText);
final newBody =
newContent.tryGet<String>('formatted_body', TryGet.silent);
if (plaintextBody &&
newContent['format'] == 'org.matrix.custom.html' &&
newBody != null &&
newBody.isNotEmpty) {
mayHaveReplyFallback = false;
body = HtmlToText.convert(newBody);
} else {
htmlMessage = false;
mayHaveReplyFallback = true;
body = newContent.tryGet<String>('body') ?? body;
}
}
// Hide reply fallback
// Be sure that the plaintextBody already stripped teh reply fallback,
// if the message is formatted
if (hideReply && (!plaintextBody || htmlMessage)) {
if (hideReply && mayHaveReplyFallback) {
body = body.replaceFirst(
RegExp(r'^>( \*)? <[^>]+>[^\n\r]+\r?\n(> [^\n]*\r?\n)*\r?\n'), '');
}

View File

@ -28,7 +28,7 @@ import 'fake_client.dart';
void main() {
/// All Tests related to the Event
group('Event', tags: 'olm', () {
group('Event', () {
Logs().level = Level.error;
final timestamp = DateTime.now().millisecondsSinceEpoch;
@ -56,6 +56,8 @@ void main() {
final event = Event.fromJson(
jsonObj, Room(id: '!testroom:example.abc', client: client));
tearDownAll(() async => client.dispose());
test('Create from json', () async {
jsonObj['content'] = json.decode(contentJson);
expect(event.toJson(), jsonObj);
@ -237,6 +239,8 @@ void main() {
expect(event.content.isEmpty, true);
redactionEventJson.remove('redacts');
expect(event.unsigned?['redacted_because'], redactionEventJson);
await client.dispose();
}
});
@ -248,6 +252,8 @@ void main() {
expect(() async => await event.cancelSend(), throwsException);
event.status = EventStatus.sending;
await event.cancelSend();
await room.client.dispose();
});
test('sendAgain', () async {
@ -269,7 +275,7 @@ void main() {
await matrix.dispose(closeDatabase: true);
});
test('requestKey', () async {
test('requestKey', tags: 'olm', () async {
final matrix = Client('testclient', httpClient: FakeMatrixApi());
await matrix.checkHomeserver(Uri.parse('https://fakeserver.notexisting'),
checkWellKnown: false);
@ -310,7 +316,7 @@ void main() {
await matrix.dispose(closeDatabase: true);
});
test('requestKey', () async {
test('requestKey', tags: 'olm', () async {
jsonObj['state_key'] = '@alice:example.com';
final event = Event.fromJson(
jsonObj, Room(id: '!localpart:server.abc', client: client));
@ -327,6 +333,8 @@ void main() {
),
);
expect(event.canRedact, true);
await client.dispose();
});
test('getLocalizedBody, isEventKnown', () async {
final matrix = Client('testclient', httpClient: FakeMatrixApi());
@ -961,6 +969,8 @@ void main() {
expect(await event.calcLocalizedBody(MatrixDefaultLocalizations()),
'Unknown event unknown.event.type');
expect(event.isEventTypeKnown, false);
await matrix.dispose(closeDatabase: true);
});
test('getLocalizedBody, parameters', () async {
@ -1126,6 +1136,8 @@ void main() {
await event.calcLocalizedBody(MatrixDefaultLocalizations()),
'Example accepted key verification request',
);
await matrix.dispose(closeDatabase: true);
});
test('aggregations', () {
@ -1200,6 +1212,581 @@ void main() {
}, room);
expect(event.plaintextBody, '**blah**');
});
test('body', () {
final event = Event.fromJson({
'type': EventTypes.Message,
'content': {
'body': 'blah',
'msgtype': 'm.text',
'format': 'org.matrix.custom.html',
'formatted_body': '<b>blub</b>',
},
'event_id': '\$source',
'sender': '@alice:example.org',
}, room);
expect(event.body, 'blah');
final event2 = Event.fromJson({
'type': EventTypes.Message,
'content': {
'body': '',
'msgtype': 'm.text',
'format': 'org.matrix.custom.html',
'formatted_body': '<b>blub</b>',
},
'event_id': '\$source',
'sender': '@alice:example.org',
}, room);
expect(event2.body, 'm.room.message');
});
group('unlocalized body reply stripping', () {
int i = 0;
void testUnlocalizedBody({
required Object? body,
required Object? formattedBody,
required bool html,
Object? editBody,
Object? editFormattedBody,
bool editHtml = false,
bool isEdit = false,
required String expectation,
required bool plaintextBody,
}) {
i += 1;
test('$i', () {
final event = Event.fromJson({
'type': EventTypes.Message,
'content': {
'msgtype': 'm.text',
if (body != null) 'body': body,
if (formattedBody != null) 'formatted_body': formattedBody,
if (html) 'format': 'org.matrix.custom.html',
if (isEdit) ...{
'm.new_content': {
if (editBody != null) 'body': editBody,
if (editFormattedBody != null)
'formatted_body': editFormattedBody,
if (editHtml) 'format': 'org.matrix.custom.html',
},
'm.relates_to': {
'event_id': '\$source2',
'rel_type': RelationshipTypes.edit,
},
},
},
'event_id': '\$source',
'sender': '@alice:example.org',
}, room);
expect(
event.calcUnlocalizedBody(
hideReply: true, hideEdit: true, plaintextBody: plaintextBody),
expectation,
reason:
'event was ${event.toJson()} and plaintextBody ${plaintextBody ? "was" : "was not"} set',
);
});
}
// everything where we expect the body to be returned
testUnlocalizedBody(
expectation: 'body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
// not sure we actually want m.room.message here and not an empty string
expectation: 'm.room.message',
plaintextBody: false,
body: '',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'm.room.message',
plaintextBody: false,
body: null,
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'm.room.message',
plaintextBody: false,
body: 5,
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: true,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: false,
body: 'body',
formattedBody: null,
html: true,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: true,
body: 'body',
formattedBody: null,
html: true,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: true,
body: 'body',
// do we actually expect this to then use the body?
formattedBody: '',
html: true,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: true,
body: 'body',
formattedBody: 5,
html: true,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: null,
editFormattedBody: null,
editHtml: true,
);
testUnlocalizedBody(
expectation: '**formatted body**',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: '**formatted body**',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: null,
editFormattedBody: null,
editHtml: true,
);
// everything where we expect the formatted body to be returned
testUnlocalizedBody(
expectation: '**formatted body**',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: '**formatted body**',
plaintextBody: true,
body: 5,
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: false,
editBody: null,
editFormattedBody: null,
editHtml: false,
);
// everything where we expect the edit body to be returned
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: true,
editBody: 'edit body',
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: false,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: true,
editBody: 'edit body',
editFormattedBody: null,
editHtml: true,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: 5,
editHtml: true,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: false,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: true,
editBody: 'edit body',
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: false,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: false,
isEdit: true,
editBody: 'edit body',
editFormattedBody: null,
editHtml: true,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: null,
editHtml: false,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: false,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: null,
editHtml: true,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: null,
editHtml: false,
);
// everything where we expect the edit formatted body to be returned
testUnlocalizedBody(
expectation: '**edit formatted body**',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: null,
editFormattedBody: '<b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: '**edit formatted body**',
plaintextBody: true,
body: 'body',
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: '**edit formatted body**',
plaintextBody: true,
body: null,
formattedBody: '<b>formatted body</b>',
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: '**edit formatted body**',
plaintextBody: true,
body: 'body',
formattedBody: null,
html: true,
isEdit: true,
editBody: 'edit body',
editFormattedBody: '<b>edit formatted body</b>',
editHtml: true,
);
// test with reply fallback
testUnlocalizedBody(
expectation: 'body',
plaintextBody: false,
body: '> <@some:user.id> acb def\n\nbody',
formattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>formatted body</b>',
html: true,
isEdit: false,
// strictly speaking there is no quote in edits, but we have to handle them anyway because of other clients doing it wrong
editBody: '> <@some:user.id> acb def\n\nedit body',
editFormattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: true,
body: '> <@some:user.id> acb def\n\nbody',
formattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>formatted body</b>',
html: false,
isEdit: false,
// strictly speaking there is no quote in edits, but we have to handle them anyway because of other clients doing it wrong
editBody: '> <@some:user.id> acb def\n\nedit body',
editFormattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: 'body',
plaintextBody: true,
body: '> <@some:user.id> acb def\n\nbody',
formattedBody: null,
html: true,
isEdit: false,
// strictly speaking there is no quote in edits, but we have to handle them anyway because of other clients doing it wrong
editBody: '> <@some:user.id> acb def\n\nedit body',
editFormattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: '**formatted body**',
plaintextBody: true,
body: '> <@some:user.id> acb def\n\nbody',
formattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>formatted body</b>',
html: true,
isEdit: false,
// strictly speaking there is no quote in edits, but we have to handle them anyway because of other clients doing it wrong
editBody: '> <@some:user.id> acb def\n\nedit body',
editFormattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: false,
body: '> <@some:user.id> acb def\n\nbody',
formattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>formatted body</b>',
html: true,
isEdit: true,
// strictly speaking there is no quote in edits, but we have to handle them anyway because of other clients doing it wrong
editBody: '> <@some:user.id> acb def\n\nedit body',
editFormattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>edit formatted body</b>',
editHtml: true,
);
testUnlocalizedBody(
expectation: 'edit body',
plaintextBody: true,
body: '> <@some:user.id> acb def\n\nbody',
formattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>formatted body</b>',
html: true,
isEdit: true,
// strictly speaking there is no quote in edits, but we have to handle them anyway because of other clients doing it wrong
editBody: '> <@some:user.id> acb def\n\nedit body',
editFormattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>edit formatted body</b>',
editHtml: false,
);
testUnlocalizedBody(
expectation: '**edit formatted body**',
plaintextBody: true,
body: '> <@some:user.id> acb def\n\nbody',
formattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>formatted body</b>',
html: true,
isEdit: true,
// strictly speaking there is no quote in edits, but we have to handle them anyway because of other clients doing it wrong
editBody: '> <@some:user.id> acb def\n\nedit body',
editFormattedBody:
'<mx-reply><blockquote>abc</blockquote></mx-reply><b>edit formatted body</b>',
editHtml: true,
);
});
test('getDisplayEvent', () {
final room = Room(id: '!1234', client: client);
var event = Event.fromJson({
@ -1394,6 +1981,7 @@ void main() {
});
test(
'encrypted attachments',
tags: 'olm',
() async {
final FILE_BUFF_ENC =
Uint8List.fromList([0x3B, 0x6B, 0xB2, 0x8C, 0xAF]);
@ -1504,7 +2092,7 @@ void main() {
await room.client.dispose(closeDatabase: true);
},
);
test('downloadAndDecryptAttachment store', () async {
test('downloadAndDecryptAttachment store', tags: 'olm', () async {
final FILE_BUFF = Uint8List.fromList([0]);
var serverHits = 0;
Future<Uint8List> downloadCallback(Uri uri) async {
@ -1546,7 +2134,7 @@ void main() {
await room.client.dispose(closeDatabase: true);
});
test('downloadAndDecryptAttachment store only', () async {
test('downloadAndDecryptAttachment store only', tags: 'olm', () async {
final FILE_BUFF = Uint8List.fromList([0]);
var serverHits = 0;
Future<Uint8List> downloadCallback(Uri uri) async {
@ -1595,7 +2183,8 @@ void main() {
await room.client.dispose(closeDatabase: true);
});
test('downloadAndDecryptAttachment store only without file', () async {
test('downloadAndDecryptAttachment store only without file', tags: 'olm',
() async {
final FILE_BUFF = Uint8List.fromList([0]);
var serverHits = 0;
Future<Uint8List> downloadCallback(Uri uri) async {