Skip to content

Commit

Permalink
ICU-22100 Improve Japanese phrase breaking performance
Browse files Browse the repository at this point in the history 
See unicode-org#2287
  • Loading branch information
@allensu05
allensu05 authored and Squash Bot committed Jan 20, 2023
1 parent cd1b772 commit aab64c8
Showing 1 changed file with 45 additions and 87 deletions.
132 changes: 45 additions & 87 deletions icu4c/source/common/mlbe.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,6 @@ MlBreakEngine::~MlBreakEngine() {}

namespace {
const char16_t INVALID = u'|';
const int32_t MAX_FEATURE = 13;
const int32_t MAX_FEATURE_LENGTH = 11;

void concatChar(const char16_t *str, const UChar32 *arr, int32_t length, char16_t *feature, UErrorCode &status) {
if (U_FAILURE(status)) {
return;
}
UnicodeString result(str);
for (int i = 0; i < length; i++) {
result.append(arr[i]);
}
U_ASSERT(result.length() < MAX_FEATURE_LENGTH);
result.extract(feature, MAX_FEATURE_LENGTH, status); // NUL-terminates
}
}

int32_t MlBreakEngine::divideUpRange(UText *inText, int32_t rangeStart, int32_t rangeEnd,
Expand Down Expand Up @@ -144,96 +130,68 @@ int32_t MlBreakEngine::divideUpRange(UText *inText, int32_t rangeStart, int32_t

void MlBreakEngine::evaluateBreakpoint(UChar32* elementList, int32_t index, int32_t &numBreaks,
UVector32 &boundary, UErrorCode &status) const {
char16_t featureList[MAX_FEATURE][MAX_FEATURE_LENGTH];
if (U_FAILURE(status)) {
return;
}

UChar32 arr[4] = {-1, -1, -1, -1};
int32_t length = 0, listLength = 0;

const UChar32 w1 = elementList[0];
const UChar32 w2 = elementList[1];
const UChar32 w3 = elementList[2];
const UChar32 w4 = elementList[3];
const UChar32 w5 = elementList[4];
const UChar32 w6 = elementList[5];
UnicodeString feature;
int32_t score = fNegativeSum;

length = 1;
if (w1 != INVALID) {
arr[0] = w1;
concatChar(u"UW1:", arr, length, featureList[listLength++], status);
if (elementList[0] != INVALID) {
// When the key doesn't exist, Hashtable.geti(key) returns 0 and 2 * 0 = 0.
// So, we can skip to check whether fModel includes key featureList[j] or not.
score += (2 * fModel.geti(feature.setTo(u"UW1:", 4).append(elementList[0])));
}
if (w2 != INVALID) {
arr[0] = w2;
concatChar(u"UW2:", arr, length, featureList[listLength++], status);
if (elementList[1] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"UW2:", 4).append(elementList[1])));
}
if (w3 != INVALID) {
arr[0] = w3;
concatChar(u"UW3:", arr, length, featureList[listLength++], status);
if (elementList[2] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"UW3:", 4).append(elementList[2])));
}
if (w4 != INVALID) {
arr[0] = w4;
concatChar(u"UW4:", arr, length, featureList[listLength++], status);
if (elementList[3] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"UW4:", 4).append(elementList[3])));
}
if (w5 != INVALID) {
arr[0] = w5;
concatChar(u"UW5:", arr, length, featureList[listLength++], status);
if (elementList[4] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"UW5:", 4).append(elementList[4])));
}
if (w6 != INVALID) {
arr[0] = w6;
concatChar(u"UW6:", arr, length, featureList[listLength++], status);
if (elementList[5] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"UW6:", 4).append(elementList[5])));
}
length = 2;
if (w2 != INVALID && w3 != INVALID) {
arr[0] = w2;
arr[1] = w3;
concatChar(u"BW1:", arr, length, featureList[listLength++], status);
if (elementList[1] != INVALID && elementList[2] != INVALID) {
score += (2 * fModel.geti(
feature.setTo(u"BW1:", 4).append(elementList[1]).append(elementList[2])));
}
if (w3 != INVALID && w4 != INVALID) {
arr[0] = w3;
arr[1] = w4;
concatChar(u"BW2:", arr, length, featureList[listLength++], status);
if (elementList[2] != INVALID && elementList[3] != INVALID) {
score += (2 * fModel.geti(
feature.setTo(u"BW2:", 4).append(elementList[2]).append(elementList[3])));
}
if (w4 != INVALID && w5 != INVALID) {
arr[0] = w4;
arr[1] = w5;
concatChar(u"BW3:", arr, length, featureList[listLength++], status);
if (elementList[3] != INVALID && elementList[4] != INVALID) {
score += (2 * fModel.geti(
feature.setTo(u"BW3:", 4).append(elementList[3]).append(elementList[4])));
}
length = 3;
if (w1 != INVALID && w2 != INVALID && w3 != INVALID) {
arr[0] = w1;
arr[1] = w2;
arr[2] = w3;
concatChar(u"TW1:", arr, length, featureList[listLength++], status);
if (elementList[0] != INVALID && elementList[1] != INVALID && elementList[2] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"TW1:", 4)
.append(elementList[0])
.append(elementList[1])
.append(elementList[2])));
}
if (w2 != INVALID && w3 != INVALID && w4 != INVALID) {
arr[0] = w2;
arr[1] = w3;
arr[2] = w4;
concatChar(u"TW2:", arr, length, featureList[listLength++], status);
if (elementList[1] != INVALID && elementList[2] != INVALID && elementList[3] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"TW2:", 4)
.append(elementList[1])
.append(elementList[2])
.append(elementList[3])));
}
if (w3 != INVALID && w4 != INVALID && w5 != INVALID) {
arr[0] = w3;
arr[1] = w4;
arr[2] = w5;
concatChar(u"TW3:", arr, length, featureList[listLength++], status);
if (elementList[2] != INVALID && elementList[3] != INVALID && elementList[4] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"TW3:", 4)
.append(elementList[2])
.append(elementList[3])
.append(elementList[4])));
}
if (w4 != INVALID && w5 != INVALID && w6 != INVALID) {
arr[0] = w4;
arr[1] = w5;
arr[2] = w6;
concatChar(u"TW4:", arr, length, featureList[listLength++], status);
}
if (U_FAILURE(status)) {
return;
}
int32_t score = fNegativeSum;
for (int32_t j = 0; j < listLength; j++) {
UnicodeString key(featureList[j]);
if (fModel.containsKey(key)) {
score += (2 * fModel.geti(key));
}
if (elementList[3] != INVALID && elementList[4] != INVALID && elementList[5] != INVALID) {
score += (2 * fModel.geti(feature.setTo(u"TW4:", 4)
.append(elementList[3])
.append(elementList[4])
.append(elementList[5])));
}
if (score > 0) {
boundary.addElement(index, status);
Expand Down

0 comments on commit aab64c8

Please sign in to comment.