build_data.js

/* eslint-disable no-console */
import fs from 'node:fs';
import { styleText } from 'node:util';
import prettyStringify from 'json-stringify-pretty-compact';
import shell from 'shelljs';
import YAML from 'js-yaml';
import { pull } from 'es-toolkit/compat';
import dotenv from 'dotenv';
import cldrTerritoryInfo from 'cldr-core/supplemental/territoryInfo.json' with { type: 'json' };
import packageJson from '../package.json' with { type: 'json' };
import * as languageNames from './language_names.js';

// fontawesome icons
import * as fontawesome from '@fortawesome/fontawesome-svg-core';
import { fas } from '@fortawesome/free-solid-svg-icons';
import { far } from '@fortawesome/free-regular-svg-icons';
import { fab } from '@fortawesome/free-brands-svg-icons';
fontawesome.library.add(fas, far, fab);

dotenv.config({ quiet: true });

const presetsVersion = packageJson.devDependencies['@openstreetmap/id-tagging-schema'];
/* eslint-disable no-process-env */
const presetsUrl = (process.env.ID_PRESETS_CDN_URL || 'https://cdn.jsdelivr.net/npm/@openstreetmap/id-tagging-schema@{presets_version}').replace('{presets_version}', presetsVersion);
/* eslint-enable no-process-env */

let _currBuild = null;


// if called directly, do the thing.
if (process.argv[1].indexOf('build_data.js') > -1) {
  buildData();
}


function buildData() {
  if (_currBuild) return _currBuild;

  const START = '🏗   ' + styleText('yellow', 'Building data...');
  const END = '👍  ' + styleText('green', 'data built');

  console.log('');
  console.log(START);
  console.time(END);

  // Create symlinks if necessary..  { 'target': 'source' }
  const symlinks = {
    img: 'dist/img'
  };

  for (let target of Object.keys(symlinks)) {
    if (!shell.test('-L', target)) {
      console.log(`Creating symlink:  ${target} -> ${symlinks[target]}`);
      shell.ln('-sf', symlinks[target], target);
    }
  }

  // Start clean
  shell.rm('-f', [
    'data/territory_languages.json',
    'dist/locales/en.json',
    'dist/data/*',
    'svg/fontawesome/*.svg',
  ]);

  // compile Font Awesome icons
  let faIcons = new Set([
    // list here the icons we want to use in the UI that aren't tied to other data
    'fas-i-cursor',
    'fas-lock',
    'fas-th-list',
    'fas-user-cog',
    'fas-calendar-days',
    'fas-rotate',
    'fas-eye-dropper'
  ]);
  // add icons for QA integrations
  readQAIssueIcons(faIcons);

  let territoryLanguages = generateTerritoryLanguages();
  fs.writeFileSync('data/territory_languages.json', prettyStringify(territoryLanguages, { maxLength: 9999 }) );

  writeEnJson();

  const languageInfo = languageNames.langNamesInNativeLang;
  fs.writeFileSync('data/languages.json', prettyStringify(languageInfo, { maxLength: 200 }));
  fs.writeFileSync('dist/data/languages.min.json', JSON.stringify(languageInfo));

  // Save individual data files
  let tasks = [
    minifyJSON('data/address_formats.json', 'dist/data/address_formats.min.json'),
    minifyJSON('data/imagery.json', 'dist/data/imagery.min.json'),
    minifyJSON('data/intro_graph.json', 'dist/data/intro_graph.min.json'),
    minifyJSON('data/languages.json', 'dist/data/languages.min.json'),
    minifyJSON('data/phone_formats.json', 'dist/data/phone_formats.min.json'),
    minifyJSON('data/qa_data.json', 'dist/data/qa_data.min.json'),
    minifyJSON('data/shortcuts.json', 'dist/data/shortcuts.min.json'),
    minifyJSON('data/territory_languages.json', 'dist/data/territory_languages.min.json'),
    Promise.all([
      // Fetch the icons that are needed by the expected tagging schema version
      fetchOrRequire(`${presetsUrl}/dist/presets.min.json`),
      fetchOrRequire(`${presetsUrl}/dist/preset_categories.min.json`),
      fetchOrRequire(`${presetsUrl}/dist/fields.min.json`)
    ])
    .then(responses => Promise.all(responses.map(response => response.json())))
    .then((results) => {
      // compile the icons used by all the presets
      results.forEach(function(data) {
        for (var key in data) {
          var datum  = data[key];
          // fontawesome icon
          if (datum.icon && /^fa[srb]-/.test(datum.icon)) {
            faIcons.add(datum.icon);
          }
          if (datum.icons) {
            Object.values(datum.icons)
              .filter(icon => /^fa[srb]-/.test(icon))
              .forEach(icon => faIcons.add(icon));
          }
        }
      });
    }).then(() =>
      // also fetch the bleeding edge data too to make sure we're always hosting the latest icons
      fetch('https://raw.githubusercontent.com/openstreetmap/id-tagging-schema/interim/icons.json')
    ).then(response => response.json()).then(cuttingEdgeIcons => {
      cuttingEdgeIcons
        .filter(icon => /^fa[srb]-/.test(icon))
        .forEach(icon => faIcons.add(icon));
    }).then(() => {
      // copy over only those Font Awesome icons that we need
      writeFaIcons(faIcons);
    })
  ];

  return _currBuild =
    Promise.all(tasks)
    .then(() => {
      console.timeEnd(END);
      console.log('');
      _currBuild = null;
    })
    .catch((err) => {
      console.error(err);
      console.log('');
      _currBuild = null;
      process.exit(1);
    });
}


function readQAIssueIcons(faIcons) {
  const qa = JSON.parse(fs.readFileSync('data/qa_data.json', 'utf8'));

  for (const service in qa) {
    for (const item in qa[service].icons) {
      const icon = qa[service].icons[item];

      // fontawesome icon, remember for later
      if (/^fa[srb]-/.test(icon)) {
        faIcons.add(icon);
      }
    }
  }
}


function generateTerritoryLanguages() {
  const allRawInfo = cldrTerritoryInfo.supplemental.territoryInfo;
  let territoryLanguages = {};

  Object.keys(allRawInfo).forEach(territoryCode => {
    let territoryLangInfo = allRawInfo[territoryCode].languagePopulation;
    if (!territoryLangInfo) return;
    let langCodes = Object.keys(territoryLangInfo);

    territoryLanguages[territoryCode.toLowerCase()] = langCodes.sort((langCode1, langCode2) => {
      let popPercent1 = parseFloat(territoryLangInfo[langCode1]._populationPercent);
      let popPercent2 = parseFloat(territoryLangInfo[langCode2]._populationPercent);
      if (popPercent1 === popPercent2) {
        return langCode1.localeCompare(langCode2, 'en', { sensitivity: 'base' });
      }
      return popPercent2 - popPercent1;
    }).map(langCode => langCode.replace('_', '-'));
  });

  // override/adjust some territory languages which are not included in CLDR data
  territoryLanguages.pk.push('pnb', 'scl', 'trw', 'kls'); // https://github.com/openstreetmap/iD/pull/9242
  pull(territoryLanguages.pk, 'pa-Arab', 'lah', 'tg-Arab'); // - " -
  territoryLanguages.au = [
     'en', 'aus', 'aer', 'aoi', 'bdy', 'coa', 'dgw', 'gjm', 'gjr', 'gup',
    'jay', 'mwf', 'mwp', 'nys', 'pih', 'piu', 'pjt', 'rop', 'tcs', 'tiw',
    'ulk', 'wbp', 'wrh', 'wth', 'wyi', 'xdk', 'xni', 'xph', 'xrd', 'zku'
  ]; // https://github.com/openstreetmap/iD/pull/10684
  territoryLanguages.nz.push('rrm'); // https://github.com/openstreetmap/iD/pull/10684

  return territoryLanguages;
}


function writeEnJson() {
  const readCoreYaml = fs.readFileSync('data/core.yaml', 'utf8');
  const readImagery = fs.readFileSync('node_modules/@openstreetmap/editor-layer-index/i18n/en.yaml', 'utf8');
  const readCommunity = fs.readFileSync('node_modules/osm-community-index/i18n/en.yaml', 'utf8');
  const readManualImagery = fs.readFileSync('data/manual_imagery.json', 'utf8');

  return Promise.all([readCoreYaml, readImagery, readCommunity, readManualImagery])
    .then(data => {
      let core = YAML.load(data[0]);
      let imagery = YAML.load(data[1]);
      let community = YAML.load(data[2]);
      let manualImagery = JSON.parse(data[3]);

      for (let i in manualImagery) {
        let layer = manualImagery[i];
        let id = layer.id;
        for (let key in layer) {
          if (key === 'attribution') {
            for (let attrKey in layer[key]) {
              if (attrKey !== 'text') {
                delete layer[key][attrKey];
              }
            }
          } else if (['name', 'description'].indexOf(key) === -1) {
            delete layer[key];
          }
        }
        // tack on strings for additional imagery not included in the index
        imagery.en.imagery[id] = layer;
      }

      let enjson = core;
      let props = ['imagery', 'community', 'languageNames', 'scriptNames'];
      props.forEach(function(prop) {
        if (enjson.en[prop]) {
          console.error(`Error: Reserved property '${prop}' already exists in core strings`);
          process.exit(1);
        }
      });

      enjson.en.imagery = imagery.en.imagery;
      enjson.en.community = community.en;
      enjson.en.languageNames = languageNames.languageNamesInLanguageOf('en');
      enjson.en.scriptNames = languageNames.scriptNamesInLanguageOf('en');

      fs.writeFileSync('dist/locales/en.min.json', JSON.stringify(enjson));
    });
}


function writeFaIcons(faIcons) {
  Array.from(faIcons).forEach(function(key) {
    const prefix = key.substring(0, 3);   // `fas`, `far`, `fab`
    const name = key.substring(4);
    const def = fontawesome.findIconDefinition({ prefix: prefix, iconName: name });
    try {
      const svg = fontawesome.icon(def, {
        attributes: { xmlns: 'http://www.w3.org/2000/svg' },
      }).html.toString();
      fs.writeFileSync(`svg/fontawesome/${key}.svg`, svg);
    } catch (error) {
      console.error(`Error: No FontAwesome icon for ${key}`);
      throw (error);
    }
  });
}


function minifyJSON(inPath, outPath) {
  return new Promise((resolve, reject) => {
    fs.readFile(inPath, 'utf8', (err, data) => {
      if (err) return reject(err);

      const minified = JSON.stringify(JSON.parse(data));
      fs.writeFile(outPath, minified, (err) => {
        if (err) return reject(err);
        resolve();
      });

    });
  });
}


function fetchOrRequire(url) {
  if (url.startsWith('.')) {
    return import(url, { with: { type: 'json' } })
      .then(module => ({ json: () => module.default }));
  } else {
    return fetch(url);
  }
}