Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CLDR-17719 update v46 charts #3800

Merged
merged 6 commits into from
Jun 13, 2024
Merged

CLDR-17719 update v46 charts #3800

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
591f6a8
CLDR-17719 update v46 charts
macchiati Jun 12, 2024
e3ec337
CLDR-17719 Update coverage goals
macchiati Jun 12, 2024
69112d2
CLDR-17719 Add reformatted TSV for coverage goals
macchiati Jun 12, 2024
178af67
CLDR-17719 fix URL
macchiati Jun 12, 2024
c8fdedd
CLDR-17719 Use and cleanup constants
macchiati Jun 12, 2024
29340d5
CLDR-17719 Fix syntax
macchiati Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tools/cldr-code/src/main/java/org/unicode/cldr/tool/Chart.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public abstract class Chart {
public static final String CHART_VERSION_DIRECTORY =
ToolConstants.getBaseDirectory(ToolConstants.CHART_VERSION);

public static final String GITHUB_ROOT = CLDRURLS.CLDR_REPO_ROOT + "/blob/main/";
public static final String GITHUB_ROOT = CLDRURLS.CLDR_REPO_MAIN;
public static final String LDML_SPEC = "https://unicode.org/reports/tr35/";

public static String dataScrapeMessage(String specPart, String testFile, String... dataFiles) {
Expand Down
5 changes: 4 additions & 1 deletion tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateTransformCharts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.unicode.cldr.util.CLDRPaths;
import org.unicode.cldr.util.CLDRTransforms;
import org.unicode.cldr.util.CLDRTransforms.ParsedTransformID;
import org.unicode.cldr.util.CLDRURLS;
import org.unicode.cldr.util.CldrUtility;
import org.unicode.cldr.util.FileCopier;
import org.unicode.cldr.util.Pair;
Expand Down Expand Up @@ -367,7 +368,9 @@ private static void showAllLatin() throws IOException {
+ "These charts do not show the available script and language transliterations that are not to/from Latin. "
+ "It also does not show other transforms that are available in CLDR, such as the casing transformations, "
+ "full-width and half-width transformations, and specialized transformations such as IPA-XSampa. "
+ "For the latest snapshot of the data files, see <a href='https://github.com/unicode-org/cldr/tree/main/common/transforms'>Transform XML Data</a>. "
+ "For the latest snapshot of the data files, see <a href='"
+ CLDRURLS.CLDR_REPO_BASE
+ "tree/main/common/transforms'>Transform XML Data</a>. "
+ "For more information, see below.</blockquote>");
index.flush();
index.println("<ul>");
Expand Down
316 changes: 167 additions & 149 deletions tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowLanguages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@

import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeMultimap;
import com.ibm.icu.impl.Relation;
import com.ibm.icu.impl.Row.R2;
Expand Down Expand Up @@ -1362,7 +1362,7 @@ private String getLanguagePluralMessage(String msg, String languageCode) {
}

private String getLanguageName(String languageCode) {
String result = english.getName(languageCode);
String result = english.getName(languageCode, true, CLDRFile.SHORT_ALTS);
if (!result.equals(languageCode)) return result;
Set<String> names = Iso639Data.getNames(languageCode);
if (names != null && names.size() != 0) {
Expand All @@ -1372,171 +1372,189 @@ private String getLanguageName(String languageCode) {
}

static final Set<Organization> TC_Vendors =
ImmutableSet.of(
Organization.apple,
Organization.google,
Organization.microsoft,
Organization.cldr);
Sets.union(
Organization.getTCOrgs(),
// This adds the CLDR org at the end of the list
Set.of(Organization.cldr));

private void showCoverageGoals(PrintWriter pw) throws IOException {
PrintWriter pw2 =
new PrintWriter(
new FormattedFileWriter(
null,
"Coverage Goals",
null
// "<p>" +
// "The following show default coverage goals for larger
// organizations. " +
// "<i>[n/a]</i> shows where there is no specific value for a
// given organization, " +
// "while <i>(...)</i> indicates that the goal is inherited from
// the parent. " +
// "A * is added if the goal differs from the parent locale's
// goal. " +
// "For information on what these goals mean (comprehensive,
// modern, moderate,...), see the LDML specification "
// +
// "<a
// href='http://www.unicode.org/reports/tr35/#Coverage_Levels'>Appendix M: Coverage Levels</a>. " +
// +
// "</p>"
,
null));

TablePrinter tablePrinter =
new TablePrinter()
// tablePrinter.setSortPriorities(0,4)
.addColumn("Language", "class='source'", null, "class='source'", false)
.setSortPriority(0)
.setBreakSpans(false)
try (PrintWriter pw2 =
new PrintWriter(
new FormattedFileWriter(
null,
"Coverage Goals",
null,
SUPPLEMENTAL_INDEX_ANCHORS));
PrintWriter coverageGoalsTsv =
FileUtilities.openUTF8Writer(
CLDRPaths.CHART_DIRECTORY + "tsv/", "coverage_goals.tsv"); ) {

TablePrinter tablePrinter =
new TablePrinter()
// tablePrinter.setSortPriorities(0,4)
.addColumn(
"Language", "class='source'", null, "class='source'", false)
.setSortPriority(0)
.setBreakSpans(false)
.addColumn(
"Code",
"class='source'",
"<a href=\""
+ CLDRURLS.CLDR_REPO_ROOT
+ "/blob/main/common/main/{0}.xml\">{0}</a>",
"class='source'",
false)
.addColumn(
"D. Votes",
"class='target'",
null,
"class='target'",
false);

Map<Organization, Map<String, Level>> vendordata = sc.getLocaleTypes();
Set<String> locales = new TreeSet<>();
Set<Organization> vendors = new LinkedHashSet<>();
Set<Organization> smallVendors = new LinkedHashSet<>();

for (Organization organization : TC_Vendors) {
// if (vendor.equals(Organization.java)) continue;
Map<String, Level> data = vendordata.get(organization);
vendors.add(organization);
tablePrinter
.addColumn(
"Code",
"class='source'",
"<a href=\"http://www.unicode.org/cldr/data/common/main/{0}.xml\">{0}</a>",
"class='source'",
organization.getDisplayName(),
"class='target'",
null,
"class='target'",
false)
.addColumn("D. Votes", "class='target'", null, "class='target'", false);
.setSpanRows(false);
locales.addAll(data.keySet());
showTabbedOrgLevels(coverageGoalsTsv, organization, data);
}

Map<Organization, Map<String, Level>> vendordata = sc.getLocaleTypes();
Set<String> locales = new TreeSet<>();
Set<Organization> vendors = new LinkedHashSet<>();
Set<Organization> smallVendors = new LinkedHashSet<>();
for (Entry<Organization, Map<String, Level>> vendorData : vendordata.entrySet()) {
Organization organization = vendorData.getKey();
if (!TC_Vendors.contains(organization)) {
smallVendors.add(organization);
Map<String, Level> data = vendordata.get(organization);
showTabbedOrgLevels(coverageGoalsTsv, organization, data);
continue;
}
}

for (Organization organization : TC_Vendors) {
// if (vendor.equals(Organization.java)) continue;
Map<String, Level> data = vendordata.get(organization);
vendors.add(organization);
tablePrinter
.addColumn(
organization.getDisplayName(),
"class='target'",
null,
"class='target'",
false)
.setSpanRows(false);
locales.addAll(data.keySet());
}
Collection<Comparable[]> data = new ArrayList<>();
List<String> list = new ArrayList<>();
LanguageTagParser ltp = new LanguageTagParser();
// String alias2 = getAlias("sh_YU");

for (Entry<Organization, Map<String, Level>> vendorData : vendordata.entrySet()) {
Organization vendor = vendorData.getKey();
if (!TC_Vendors.contains(vendor)) {
smallVendors.add(vendor);
continue;
}
}
pw2.append("<h2>TC Orgs</h2>");

Collection<Comparable[]> data = new ArrayList<>();
List<String> list = new ArrayList<>();
LanguageTagParser ltp = new LanguageTagParser();
// String alias2 = getAlias("sh_YU");

pw2.append("<h2>TC Orgs</h2>");

for (String locale : locales) {
list.clear();
String localeCode = locale.equals("*") ? "und" : locale;
String alias = getAlias(localeCode);
if (!alias.equals(localeCode)) {
throw new IllegalArgumentException(
"Should use canonical form: " + locale + " => " + alias);
}
String baseLang = ltp.set(localeCode).getLanguage();
String baseLangName = getLanguageName(baseLang);
list.add("und".equals(localeCode) ? "other" : baseLangName);
list.add(locale);
int defaultVotes =
supplementalDataInfo.getRequiredVotes(CLDRLocale.getInstance(locale), null);
list.add(String.valueOf(defaultVotes));
for (Organization vendor : vendors) {
String status = getVendorStatus(locale, vendor, vendordata);
// if (!baseLang.equals(locale) && !status.startsWith("<")) {
// String langStatus = getVendorStatus(baseLang, vendor,
// vendordata);
// if (!langStatus.equals(status)) {
// status += "*";
// }
// }
list.add(status);
for (String locale : locales) {
list.clear();
String localeCode = locale.equals("*") ? "und" : locale;
String alias = getAlias(localeCode);
if (!alias.equals(localeCode)) {
throw new IllegalArgumentException(
"Should use canonical form: " + locale + " => " + alias);
}
// String baseLang = ltp.set(localeCode).getLanguage();
String baseLangName = getLanguageName(localeCode);
list.add("und".equals(localeCode) ? "other" : baseLangName);
list.add(locale);
int defaultVotes =
supplementalDataInfo.getRequiredVotes(
CLDRLocale.getInstance(locale), null);
list.add(String.valueOf(defaultVotes));
for (Organization vendor : vendors) {
String status = getVendorStatus(locale, vendor, vendordata);
// if (!baseLang.equals(locale) &&
// !status.startsWith("<")) {
// String langStatus = getVendorStatus(baseLang,
// vendor,
// vendordata);
// if (!langStatus.equals(status)) {
// status += "*";
// }
// }
list.add(status);
}
data.add(list.toArray(new String[list.size()]));
}
data.add(list.toArray(new String[list.size()]));
}
Comparable[][] flattened = data.toArray(new Comparable[data.size()][]);
String value = tablePrinter.addRows(flattened).toTable();
pw2.println(value);

pw2.append("<h2>Others</h2><div align='left'><ul>");

for (Organization vendor2 : smallVendors) {
pw2.append("<li><b>");
pw2.append(TransliteratorUtilities.toHTML.transform(vendor2.getDisplayName()))
.append(": </b>");
boolean first1 = true;
for (Level level : Level.values()) {
boolean first2 = true;
Level other = null;
for (Entry<String, Level> data2 : vendordata.get(vendor2).entrySet()) {
String key = data2.getKey();
Level level2 = data2.getValue();
if (level != level2) {
continue;
}
if (key.equals("*")) {
other = level2;
continue;
}
if (first2) {
if (first1) {
first1 = false;
Comparable[][] flattened = data.toArray(new Comparable[data.size()][]);
String value = tablePrinter.addRows(flattened).toTable();
pw2.println(value);

pw2.append("<h2>Others</h2><div align='left'><ul>");

for (Organization vendor2 : smallVendors) {
pw2.append("<li><b>");
pw2.append(TransliteratorUtilities.toHTML.transform(vendor2.getDisplayName()))
.append(": </b>");
boolean first1 = true;
for (Level level : Level.values()) {
boolean first2 = true;
Level other = null;
for (Entry<String, Level> data2 : vendordata.get(vendor2).entrySet()) {
String key = data2.getKey();
Level level2 = data2.getValue();
if (level != level2) {
continue;
}
if (key.equals("*")) {
other = level2;
continue;
}
if (first2) {
if (first1) {
first1 = false;
} else {
pw2.append("; ");
}
pw2.append(level2.toString()).append(": ");
first2 = false;
} else {
pw2.append("; ");
pw2.append(", ");
}
pw2.append(level2.toString()).append(": ");
first2 = false;
} else {
pw2.append(", ");
pw2.append(TransliteratorUtilities.toHTML.transform(key));
}
pw2.append(TransliteratorUtilities.toHTML.transform(key));
}
if (other != null) {
if (first2) {
if (first1) {
first1 = false;
if (other != null) {
if (first2) {
if (first1) {
first1 = false;
} else {
pw2.append("; ");
}
pw2.append(level.toString()).append(": ");
first2 = false;
} else {
pw2.append("; ");
pw2.append(", ");
}
pw2.append(level.toString()).append(": ");
first2 = false;
} else {
pw2.append(", ");
pw2.append("<i>other</i>");
}
pw2.append("<i>other</i>");
}
pw2.append("</li>");
}
pw2.append("</li>");
pw2.append("</ul></div>");
}
}

public void showTabbedOrgLevels(
PrintWriter coverageGoalsTsv, Organization organization, Map<String, Level> data) {
coverageGoalsTsv.println(
String.format(
"\n#%s\t;\t%s\t;\t%s\t;\t%s\n",
"Org", "Locale", "Level", "Locale Name"));
for (Entry<String, Level> entry : data.entrySet()) {
String locale = entry.getKey();
Level level = entry.getValue();
final String name =
locale.equals("*")
? "ALL"
: english.getName(locale, true, CLDRFile.SHORT_ALTS);
coverageGoalsTsv.println(
String.format(
"%s\t;\t%s\t;\t%s\t;\t%s", organization, locale, level, name));
}
pw2.append("</ul></div>");
pw2.close();
}

LanguageTagParser lpt2 = new LanguageTagParser();
Expand Down
Loading
Loading