Pre-fetch all Locize translations as app-wide static data (GCMS), bas…

…ed on #337 (#338)

src/layouts/core/coreLayoutSSG.ts

@@ -14,18 +14,17 @@ import {
   StaticCustomer,
   StaticDataset,
 } from '@/modules/core/gql/fetchGraphcmsDataset';
-import { getSharedGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
+import { getStaticGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
 import { prepareGraphCMSLocaleHeader } from '@/modules/core/gql/graphcms';
+import { getStaticLocizeTranslations } from '@/modules/core/i18n/getLocizeTranslations';
 import {
   DEFAULT_LOCALE,
   resolveFallbackLanguage,
 } from '@/modules/core/i18n/i18n';
-import { supportedLocales } from '@/modules/core/i18n/i18nConfig';
 import {
   fetchTranslations,
   I18nextResources,
 } from '@/modules/core/i18n/i18nextLocize';
-import { I18nLocale } from '@/modules/core/i18n/types/I18nLocale';
 import { createLogger } from '@/modules/core/logging/logger';
 import { PreviewData } from '@/modules/core/previewMode/types/PreviewData';
 import serializeSafe from '@/modules/core/serializeSafe/serializeSafe';
@@ -64,11 +63,11 @@ const logger = createLogger({
  * @see https://nextjs.org/docs/basic-features/data-fetching#getstaticpaths-static-generation
  */
 export const getCoreStaticPaths: GetStaticPaths<CommonServerSideParams> = async (context: GetStaticPathsContext): Promise<StaticPathsOutput> => {
-  const sharedDataset: StaticDataset = await getSharedGraphcmsDataset();
-  const sharedCustomer: StaticCustomer = sharedDataset?.customer;
+  const staticDataset: StaticDataset = await getStaticGraphcmsDataset();
+  const staticCustomer: StaticCustomer = staticDataset?.customer;
 
   // Generate only pages for languages that have been allowed by the customer
-  const paths: StaticPath[] = map(sharedCustomer?.availableLanguages, (availableLanguage: string): StaticPath => {
+  const paths: StaticPath[] = map(staticCustomer?.availableLanguages, (availableLanguage: string): StaticPath => {
     return {
       params: {
         locale: availableLanguage,
@@ -108,7 +107,6 @@ export const getCoreStaticProps: GetStaticProps<SSGPageProps, CommonServerSidePa
   const lang: string = locale.split('-')?.[0];
   const bestCountryCodes: string[] = [lang, resolveFallbackLanguage(lang)];
   const gcmsLocales: string = prepareGraphCMSLocaleHeader(bestCountryCodes);
-  const i18nTranslations: I18nextResources = await fetchTranslations(lang); // Pre-fetches translations from Locize API
   const apolloClient: ApolloClient<NormalizedCacheObject> = initializeApollo();
   const variables = {
     customerRef,
@@ -147,6 +145,16 @@ export const getCoreStaticProps: GetStaticProps<SSGPageProps, CommonServerSidePa
     customer,
   };
 
+  let i18nTranslations: I18nextResources;
+
+  if (preview) {
+    // When preview mode is enabled, we want to make real-time API requests to get up-to-date data
+    i18nTranslations = await fetchTranslations(lang);
+  } else {
+    // When preview mode is not enabled, we fallback to the app-wide shared/static data (stale)
+    i18nTranslations = await getStaticLocizeTranslations(lang);
+  }
+
   return {
     // Props returned here will be available as page properties (pageProps)
     props: {

src/layouts/core/coreLayoutSSR.ts

@@ -10,7 +10,7 @@ import {
   StaticCustomer,
   StaticDataset,
 } from '@/modules/core/gql/fetchGraphcmsDataset';
-import { getSharedGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
+import { getStaticGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
 import { prepareGraphCMSLocaleHeader } from '@/modules/core/gql/graphcms';
 import { ApolloQueryOptions } from '@/modules/core/gql/types/ApolloQueryOptions';
 import {
@@ -125,12 +125,12 @@ export const getCoreServerSideProps: GetServerSideProps<GetCoreServerSidePropsRe
     },
   };
 
-  const sharedDataset: StaticDataset = await getSharedGraphcmsDataset();
-  const sharedCustomer: StaticCustomer = sharedDataset?.customer;
+  const staticDataset: StaticDataset = await getStaticGraphcmsDataset();
+  const staticCustomer: StaticCustomer = staticDataset?.customer;
 
   // Do not serve pages using locales the customer doesn't have enabled
-  if (!includes(sharedCustomer?.availableLanguages, locale)) {
-    logger.warn(`Locale "${locale}" not enabled for this customer (allowed: "${sharedCustomer?.availableLanguages}"), returning 404 page.`);
+  if (!includes(staticCustomer?.availableLanguages, locale)) {
+    logger.warn(`Locale "${locale}" not enabled for this customer (allowed: "${staticCustomer?.availableLanguages}"), returning 404 page.`);
 
     return {
       notFound: true,

src/layouts/demo/demoLayoutSSG.ts

@@ -14,18 +14,17 @@ import {
   StaticCustomer,
   StaticDataset,
 } from '@/modules/core/gql/fetchGraphcmsDataset';
-import { getSharedGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
+import { getStaticGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
 import { prepareGraphCMSLocaleHeader } from '@/modules/core/gql/graphcms';
+import { getStaticLocizeTranslations } from '@/modules/core/i18n/getLocizeTranslations';
 import {
   DEFAULT_LOCALE,
   resolveFallbackLanguage,
 } from '@/modules/core/i18n/i18n';
-import { supportedLocales } from '@/modules/core/i18n/i18nConfig';
 import {
   fetchTranslations,
   I18nextResources,
 } from '@/modules/core/i18n/i18nextLocize';
-import { I18nLocale } from '@/modules/core/i18n/types/I18nLocale';
 import { createLogger } from '@/modules/core/logging/logger';
 import { PreviewData } from '@/modules/core/previewMode/types/PreviewData';
 import serializeSafe from '@/modules/core/serializeSafe/serializeSafe';
@@ -64,11 +63,11 @@ const logger = createLogger({
  * @see https://nextjs.org/docs/basic-features/data-fetching#getstaticpaths-static-generation
  */
 export const getDemoStaticPaths: GetStaticPaths<CommonServerSideParams> = async (context: GetStaticPathsContext): Promise<StaticPathsOutput> => {
-  const sharedDataset: StaticDataset = await getSharedGraphcmsDataset();
-  const sharedCustomer: StaticCustomer = sharedDataset?.customer;
+  const staticDataset: StaticDataset = await getStaticGraphcmsDataset();
+  const staticCustomer: StaticCustomer = staticDataset?.customer;
 
   // Generate only pages for languages that have been allowed by the customer
-  const paths: StaticPath[] = map(sharedCustomer?.availableLanguages, (availableLanguage: string): StaticPath => {
+  const paths: StaticPath[] = map(staticCustomer?.availableLanguages, (availableLanguage: string): StaticPath => {
     return {
       params: {
         locale: availableLanguage,
@@ -108,7 +107,6 @@ export const getDemoStaticProps: GetStaticProps<SSGPageProps, CommonServerSidePa
   const lang: string = locale.split('-')?.[0];
   const bestCountryCodes: string[] = [lang, resolveFallbackLanguage(lang)];
   const gcmsLocales: string = prepareGraphCMSLocaleHeader(bestCountryCodes);
-  const i18nTranslations: I18nextResources = await fetchTranslations(lang); // Pre-fetches translations from Locize API
   const apolloClient: ApolloClient<NormalizedCacheObject> = initializeApollo();
   const variables = {
     customerRef,
@@ -147,6 +145,16 @@ export const getDemoStaticProps: GetStaticProps<SSGPageProps, CommonServerSidePa
     customer,
   };
 
+  let i18nTranslations: I18nextResources;
+
+  if (preview) {
+    // When preview mode is enabled, we want to make real-time API requests to get up-to-date data
+    i18nTranslations = await fetchTranslations(lang);
+  } else {
+    // When preview mode is not enabled, we fallback to the app-wide shared/static data (stale)
+    i18nTranslations = await getStaticLocizeTranslations(lang);
+  }
+
   return {
     // Props returned here will be available as page properties (pageProps)
     props: {

src/layouts/demo/demoLayoutSSR.ts

@@ -10,7 +10,7 @@ import {
   StaticCustomer,
   StaticDataset,
 } from '@/modules/core/gql/fetchGraphcmsDataset';
-import { getSharedGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
+import { getStaticGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
 import { prepareGraphCMSLocaleHeader } from '@/modules/core/gql/graphcms';
 import { ApolloQueryOptions } from '@/modules/core/gql/types/ApolloQueryOptions';
 import {
@@ -125,12 +125,12 @@ export const getDemoServerSideProps: GetServerSideProps<GetDemoServerSidePropsRe
     },
   };
 
-  const sharedDataset: StaticDataset = await getSharedGraphcmsDataset();
-  const sharedCustomer: StaticCustomer = sharedDataset?.customer;
+  const staticDataset: StaticDataset = await getStaticGraphcmsDataset();
+  const staticCustomer: StaticCustomer = staticDataset?.customer;
 
   // Do not serve pages using locales the customer doesn't have enabled
-  if (!includes(sharedCustomer?.availableLanguages, locale)) {
-    logger.warn(`Locale "${locale}" not enabled for this customer (allowed: "${sharedCustomer?.availableLanguages}"), returning 404 page.`);
+  if (!includes(staticCustomer?.availableLanguages, locale)) {
+    logger.warn(`Locale "${locale}" not enabled for this customer (allowed: "${staticCustomer?.availableLanguages}"), returning 404 page.`);
 
     return {
       notFound: true,

src/modules/core/gql/fetchGraphcmsDataset.preval.ts

@@ -25,8 +25,7 @@ import preval from 'next-plugin-preval';
  * XXX The data are therefore STALE, they're not fetched in real-time.
  *  They won't update (the app won't display up-to-date data until the next deployment, for static pages).
  *
- * @example await import('@/modules/core/airtable/fetchRawAirtableDataset.preval')
- * @example const rawAirtableRecordsSets: RawAirtableRecordsSet[] = (await import('@/modules/core/airtable/fetchRawAirtableDataset.preval')) as unknown as RawAirtableRecordsSet[];
+ * @example const dataset: StaticDataset = await getStaticGraphcmsDataset();
  *
  * @see https://github.com/ricokahler/next-plugin-preval
  */

src/modules/core/gql/getGraphcmsDataset.ts

@@ -12,8 +12,8 @@ const logger = createLogger({
  * This dataset is STALE. It will not update, ever.
  * The dataset is created at build time, using the "next-plugin-preval" webpack plugin.
  *
- * @example const dataset: StaticDataset = await getSharedGraphcmsDataset();
+ * @example const dataset: StaticDataset = await getStaticGraphcmsDataset();
  */
-export const getSharedGraphcmsDataset = async (): Promise<StaticDataset> => {
+export const getStaticGraphcmsDataset = async (): Promise<StaticDataset> => {
   return (await import('@/modules/core/gql/fetchGraphcmsDataset.preval')) as unknown as StaticDataset;
 };

src/modules/core/i18n/fetchLocizeTranslations.preval.ts

@@ -0,0 +1,31 @@
+import fetchLocizeTranslations from '@/modules/core/i18n/fetchLocizeTranslations';
+import preval from 'next-plugin-preval';
+
+/**
+ * Pre-fetches the Locize translations for all languages and stores the result in an cached internal JSON file.
+ * Overall, this approach allows us to have some static app-wide data that will never update, and have real-time data wherever we want.
+ *
+ * This is very useful to avoid fetching the same data for each page during the build step.
+ * By default, Next.js would call the Locize API once per page built.
+ * This was a huge pain for many reasons, because our app uses mostly static pages and we don't want those static pages to be updated.
+ *
+ * Also, even considering built time only, it was very inefficient, because Next was triggering too many API calls:
+ * - More than 40 fetch attempts (40+ demo pages)
+ * - Our in-memory cache was helping but wouldn't completely conceal the over-fetching caused by Next.js
+ * - Locize API has on-demand pricing, each call costs us money
+ *
+ * The shared/static dataset is accessible to:
+ * - All components
+ * - All pages (both getStaticProps and getStaticPaths, and even in getServerSideProps is you wish to!)
+ * - All API endpoints
+ *
+ * XXX The data are therefore STALE, they're not fetched in real-time.
+ *  They won't update (the app won't display up-to-date data until the next deployment, for static pages).
+ *
+ * @example const allStaticLocizeTranslations = await getAllStaticLocizeTranslations();
+ *
+ * @see https://github.com/ricokahler/next-plugin-preval
+ */
+export const locizeTranslations = preval(fetchLocizeTranslations());
+
+export default locizeTranslations;

src/modules/core/i18n/fetchLocizeTranslations.ts

@@ -0,0 +1,42 @@
+import { supportedLocales } from '@/modules/core/i18n/i18nConfig';
+import {
+  fetchTranslations,
+  I18nextResources,
+} from '@/modules/core/i18n/i18nextLocize';
+import { I18nLocale } from '@/modules/core/i18n/types/I18nLocale';
+import { createLogger } from '@/modules/core/logging/logger';
+
+const fileLabel = 'modules/core/i18n/fetchLocizeTranslations';
+const logger = createLogger({
+  fileLabel,
+});
+
+export type LocizeTranslationByLang = {
+  [lang: string]: I18nextResources;
+}
+
+/**
+ * Fetches the Locize API.
+ * Invoked by fetchLocizeTranslations.preval.preval.ts file at build time (during Webpack bundling).
+ *
+ * XXX Must be a single export file otherwise it can cause issues - See https://github.com/ricokahler/next-plugin-preval/issues/19#issuecomment-848799473
+ *
+ * XXX We opinionately decided to use the "lang" (e.g: 'en') as Locize index, but it could also be the "name" (e.g: 'en-US'), it depends on your business requirements!
+ *  (lang is simpler)
+ */
+export const fetchLocizeTranslations = async (): Promise<LocizeTranslationByLang> => {
+  const translationsByLocale: LocizeTranslationByLang = {};
+  const promises: Promise<any>[] = [];
+
+  supportedLocales.map((supportedLocale: I18nLocale) => {
+    promises.push(fetchTranslations(supportedLocale?.lang));
+  });
+
+  // Run all promises in parallel and compute results into the dataset
+  const results: I18nextResources[] = await Promise.all(promises);
+  results.map((i18nextResources: I18nextResources, index) => translationsByLocale[supportedLocales[index]?.lang] = i18nextResources);
+
+  return translationsByLocale;
+};
+
+export default fetchLocizeTranslations;

src/modules/core/i18n/getLocizeTranslations.ts

@@ -0,0 +1,30 @@
+import { LocizeTranslationByLang } from '@/modules/core/i18n/fetchLocizeTranslations';
+import { I18nextResources } from '@/modules/core/i18n/i18nextLocize';
+import { createLogger } from '@/modules/core/logging/logger';
+
+const fileLabel = 'modules/core/i18n/getLocizeTranslations';
+const logger = createLogger({
+  fileLabel,
+});
+
+/**
+ * Returns all translations (indexed by language), based on the app-wide static/shared/stale data fetched at build time.
+ *
+ * @example const allStaticLocizeTranslations = await getAllStaticLocizeTranslations();
+ */
+export const getAllStaticLocizeTranslations = async (): Promise<LocizeTranslationByLang> => {
+  return (await import('@/modules/core/i18n/fetchLocizeTranslations.preval')) as unknown as LocizeTranslationByLang;
+};
+
+/**
+ * Returns all translations for one language, based on the app-wide static/shared/stale data fetched at build time.
+ *
+ * @example const i18nTranslations: I18nextResources = await getStaticLocizeTranslations(lang);
+ *
+ * @param lang
+ */
+export const getStaticLocizeTranslations = async (lang: string): Promise<I18nextResources> => {
+  const allStaticLocizeTranslations = await getAllStaticLocizeTranslations();
+
+  return allStaticLocizeTranslations?.[lang];
+};

src/modules/core/i18n/i18nextLocize.ts

@@ -253,7 +253,7 @@ export const fetchBaseTranslations = async (lang: string): Promise<I18nextResour
 };
 
 /**
- * Fetches the translations that are specific to the customer (its own translations variation)
+ * Fetches the translations that are specific to the customer (their own translations variation)
  *
  * @param lang
  */

src/modules/core/i18n/middlewares/localeMiddleware.ts

@@ -2,7 +2,7 @@ import {
   StaticCustomer,
   StaticDataset,
 } from '@/modules/core/gql/fetchGraphcmsDataset';
-import { getSharedGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
+import { getStaticGraphcmsDataset } from '@/modules/core/gql/getGraphcmsDataset';
 import { createLogger } from '@/modules/core/logging/logger';
 import redirect from '@/utils/redirect';
 import includes from 'lodash.includes';
@@ -35,14 +35,14 @@ export const localeMiddleware = async (req: NextApiRequest, res: NextApiResponse
   logger.debug('Detecting browser locale...');
   const detections: string[] = acceptLanguageHeaderLookup(req) || [];
   let localeFound; // Will contain the most preferred browser locale (e.g: fr-FR, fr, en-US, en, etc.)
-  const sharedDataset: StaticDataset = await getSharedGraphcmsDataset();
-  const sharedCustomer: StaticCustomer = sharedDataset?.customer;
+  const staticDataset: StaticDataset = await getStaticGraphcmsDataset();
+  const staticCustomer: StaticCustomer = staticDataset?.customer;
 
   if (detections && !!size(detections)) {
     detections.forEach((language) => {
       if (localeFound || typeof language !== 'string') return;
 
-      if (includes(sharedCustomer?.availableLanguages, language)) {
+      if (includes(staticCustomer?.availableLanguages, language)) {
         localeFound = language;
       }
     });
@@ -53,7 +53,7 @@ export const localeMiddleware = async (req: NextApiRequest, res: NextApiResponse
   }
 
   if (!localeFound) {
-    localeFound = sharedCustomer?.availableLanguages?.[0] || DEFAULT_LOCALE;
+    localeFound = staticCustomer?.availableLanguages?.[0] || DEFAULT_LOCALE;
   }
 
   logger.debug(`Locale applied: "${localeFound}", for url "${req.url}"`);