upgrade transifex to API v3

see also openstreetmap/iD#9375

Author: tyrasd

lib/translations.js

@@ -1,29 +1,29 @@
 /* Downloads the latest translations from Transifex */
 import fs from 'fs';
 import fetch from 'node-fetch';
-import btoa from 'btoa';
 import YAML from 'js-yaml';
+import { transifexApi } from '@transifex/api';
 
 
 function fetchTranslations(options) {
 
-  let defaultCredentials = {
-    user: 'api',
-    password: ''
-  };
-  if (fs.existsSync(`${process.cwd()}/transifex.auth`)) {
-    // Credentials can be stored in transifex.auth as a json object. You should probably gitignore this file.
-    // You can use an API key instead of your password: https://docs.transifex.com/api/introduction#authentication
+  // Transifex doesn't allow anonymous downloading
+  /* eslint-disable no-process-env */
+  if (process.env.transifex_password) {
+    // Deployment scripts may prefer environment variables
+    transifexApi.setup({ auth: process.env.transifex_password });
+  } else {
+    // Credentials can be stored in transifex.auth as a json object. This file is gitignored.
+    // You must use an API token for authentication: You can generate one at https://www.transifex.com/user/settings/api/.
     // {
-    //   "user": "username",
-    //   "password": "password"
+    //   "password": "<api_key>"
     // }
-    defaultCredentials = JSON.parse(fs.readFileSync(`${process.cwd()}/transifex.auth`, 'utf8'));
+    transifexApi.setup({ auth: JSON.parse(fs.readFileSync('./transifex.auth', 'utf8')).password });
   }
+  /* eslint-enable no-process-env */
 
   if (!options) options = {};
   options = Object.assign({
-    translCredentials: defaultCredentials,
     translOrgId: '',
     translProjectId: '',
     translResourceIds: ['presets'],
@@ -38,15 +38,6 @@ function fetchTranslations(options) {
     fs.mkdirSync(outDir);
   }
 
-  const fetchOpts = {
-    headers: {
-      'Authorization': 'Basic ' + btoa(options.translCredentials.user + ':' + options.translCredentials.password),
-    }
-  };
-
-  const apiroot = 'https://www.transifex.com/api/2';
-  const projectURL = `${apiroot}/project/${options.translProjectId}`;
-
   const translResourceIds = options.translResourceIds;
   return new Promise(function(resolve) {
       asyncMap(translResourceIds, getResourceInfo, function(err, results) {
@@ -59,37 +50,41 @@ function fetchTranslations(options) {
   });
 
 
-  function getResourceInfo(resourceId, callback) {
-    let url = `https://api.transifex.com/organizations/${options.translOrgId}/projects/${options.translProjectId}/resources/${resourceId}`;
-    fetch(url, fetchOpts)
-      .then(res => {
-        process.stdout.write(`${res.status}: ${url}\n`);
-        return res.json();
-      })
-      .then(json => {
-        callback(null, json);
-      })
-      .catch(err => callback(err));
+  async function getResourceInfo(resourceId, callback) {
+    try {
+      const result = [];
+      for await (const stat of transifexApi.ResourceLanguageStats.filter({
+        project: `o:${options.translOrgId}:p:${options.translProjectId}`,
+        resource: `o:${options.translOrgId}:p:${options.translProjectId}:r:${resourceId}`
+      }).all()) {
+        result.push(stat);
+      }
+      console.log(`got resource language stats collection for ${resourceId}`);
+      callback(null, result);
+    } catch (err) {
+      console.error(`error while getting resource language stats collection for ${resourceId}`, err);
+      callback(err);
+    }
   }
 
   function gotResourceInfo(err, results) {
     if (err) return process.stderr.write(err + '\n');
 
     let coverageByLocaleCode = {};
     results.forEach(function(info) {
-      for (let code in info.stats) {
-        let type = 'translated';
-        if (options.translReviewedOnly &&
-          (!Array.isArray(options.translReviewedOnly) || options.translReviewedOnly.indexOf(code) !== -1)) {
-          // reviewed_1 = reviewed, reviewed_2 = proofread
-          type = 'reviewed_1';
+      info.forEach(stat => {
+        let code = stat.relationships.language.data.id.substr(2).replace(/_/g, '-');
+        let type = 'translated_strings';
+        if (options.translReviewedOnly && (
+          !Array.isArray(options.translReviewedOnly)
+          || options.translReviewedOnly.indexOf(code) !== -1)) {
+          type = 'reviewed_strings';
         }
-        let coveragePart = info.stats[code][type].percentage / results.length;
+        let coveragePart = (stat.attributes[type] /  stat.attributes.total_strings) / results.length;
 
-        code = code.replace(/_/g, '-');
         if (coverageByLocaleCode[code] === undefined) coverageByLocaleCode[code] = 0;
         coverageByLocaleCode[code] += coveragePart;
-      }
+      });
     });
     let dataLocales = {};
     // explicitly list the source locale as having 100% coverage
@@ -112,11 +107,10 @@ function fetchTranslations(options) {
   }
 
   function getResource(resourceId, callback) {
-    let resourceURL = `${projectURL}/resource/${resourceId}`;
-    getLanguages(resourceURL, (err, codes) => {
+    getLanguages((err, codes) => {
       if (err) return callback(err);
 
-      asyncMap(codes, getLanguage(resourceURL), (err, results) => {
+      asyncMap(codes, getLanguage(resourceId), (err, results) => {
         if (err) return callback(err);
 
         let locale = {};
@@ -216,44 +210,48 @@ function fetchTranslations(options) {
     }
   }
 
-
-  function getLanguage(resourceURL) {
-    return (code, callback) => {
-      code = code.replace(/-/g, '_');
-      let url = `${resourceURL}/translation/${code}`;
-      if (options.translReviewedOnly &&
-        (!Array.isArray(options.translReviewedOnly) || options.translReviewedOnly.indexOf(code) !== -1)) {
-
-        url += '?mode=reviewed';
+  function getLanguage(resourceId) {
+    return async (code, callback) => {
+      try {
+        code = code.replace(/-/g, '_');
+        let reviewedOnly = options.translReviewedOnly && (
+          !Array.isArray(options.translReviewedOnly)
+          || options.translReviewedOnly.indexOf(code) !== -1);
+        const url = await transifexApi.ResourceTranslationsAsyncDownload.download({
+          resource: {data:{type:'resources', id:`o:${options.translOrgId}:p:${options.translProjectId}:r:${resourceId}`}},
+          language: {data:{type:'languages', id:`l:${code}`}},
+          // fetch only reviewed strings for some languages
+          mode: reviewedOnly ? 'reviewed' : 'default'
+        });
+        const data = await fetch(url).then(d => d.text());
+        process.stdout.write(`got translations for ${resourceId}, language ${code}\n`);
+        callback(null, YAML.load(data)[code]);
+      } catch (err) {
+        process.stderr.write(`error while getting translations for ${resourceId}, language ${code}\n`);
+        callback(err);
       }
-      fetch(url, fetchOpts)
-        .then(res => {
-          process.stdout.write(`${res.status}: ${url}\n`);
-          return res.json();
-        })
-        .then(json => {
-          callback(null, YAML.load(json.content)[code]);
-        })
-        .catch(err => callback(err));
     };
   }
 
 
-  function getLanguages(resourceURL, callback) {
-    let url = `${resourceURL}?details`;
-    fetch(url, fetchOpts)
-      .then(res => {
-        process.stdout.write(`${res.status}: ${url}\n`);
-        return res.json();
-      })
-      .then(json => {
-        callback(null, json.available_languages
-          .map(d => d.code.replace(/_/g, '-'))
-          // we already have the source locale so don't download it
-          .filter(d => d !== options.sourceLocale)
-        );
-      })
-      .catch(err => callback(err));
+  async function getLanguages(callback) {
+    try {
+      const result = [];
+      const project = await transifexApi.Project.get({
+        organization: `o:${options.translOrgId}`,
+        slug: options.translProjectId
+      });
+      const lngs = await project.fetch('languages');
+      for await (const lng of lngs.all()) {
+        if (lng.attributes.code === 'en') continue;
+        result.push(lng.attributes.code.replace(/_/g, '-'));
+      }
+      process.stdout.write('got project languages\n');
+      callback(null, result);
+    } catch (err) {
+      process.stderr.write('error while getting project languages\n');
+      callback(err);
+    }
   }
 }
 
@@ -269,7 +267,7 @@ function asyncMap(inputs, func, callback) {
   function next() {
     callFunc(index++);
     if (index < inputs.length) {
-      setTimeout(next, 200);
+      setTimeout(next, 50);
     }
   }
 

package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "@ideditor/schema-builder",
-  "version": "5.1.1",
+  "version": "5.2.0-dev",
   "description": "Framework for defining iD-compatible tagging models",
   "homepage": "https://github.com/ideditor/schema-builder#readme",
   "bugs": "https://github.com/ideditor/schema-builder/issues",
@@ -12,7 +12,7 @@
   "license": "ISC",
   "exports": "./lib/index.js",
   "dependencies": {
-    "btoa": "^1.2.1",
+    "@transifex/api": "^4.2.5",
     "chalk": "^5.0.1",
     "glob": "^8.0.3",
     "js-yaml": "^4.0.0",