Merge pull request #504 from atmire/i18n-sync-files-script

I18n sync script

package.json

@@ -70,7 +70,8 @@
     "docs": "typedoc --options typedoc.json ./src/",
     "coverage": "http-server -c-1 -o -p 9875 ./coverage",
     "postinstall": "yarn run patch-protractor",
-    "patch-protractor": "ncp node_modules/webdriver-manager node_modules/protractor/node_modules/webdriver-manager"
+    "patch-protractor": "ncp node_modules/webdriver-manager node_modules/protractor/node_modules/webdriver-manager",
+    "sync-i18n": "node ./scripts/sync-i18n-files.js"
   },
   "dependencies": {
     "@angular/animations": "^6.1.4",
@@ -174,7 +175,9 @@
     "angular2-template-loader": "0.6.2",
     "autoprefixer": "^9.1.3",
     "caniuse-lite": "^1.0.30000697",
+    "cli-progress": "^3.3.1",
     "codelyzer": "^4.4.4",
+    "commander": "^3.0.2",
     "compression-webpack-plugin": "^1.1.6",
     "copy-webpack-plugin": "^4.4.1",
     "copyfiles": "^2.1.1",

scripts/sync-i18n-files.js

@@ -0,0 +1,342 @@
+#!/usr/bin/env node
+const commander = require('commander');
+const fs = require('fs');
+const JSON5 = require('json5');
+const _cliProgress = require('cli-progress');
+const _ = require('lodash');
+const {projectRoot} = require('../webpack/helpers');
+
+const program = new commander.Command();
+program.version('1.0.0', '-v, --version');
+
+const NEW_MESSAGE_TODO = '// TODO New key - Add a translation';
+const MESSAGE_CHANGED_TODO = '// TODO Source message changed - Revise the translation';
+const COMMENTS_CHANGED_TODO = '// TODO Source comments changed - Revise the translation';
+
+const DEFAULT_SOURCE_FILE_LOCATION = 'resources/i18n/en.json5';
+const LANGUAGE_FILES_LOCATION = 'resources/i18n';
+
+parseCliInput();
+
+/**
+ * Parses the CLI input given by the user
+ *    If no parameters are set (standard usage) -> source file is default (set to DEFAULT_SOURCE_FILE_LOCATION) and all
+ *            other language files in the LANGUAGE_FILES_LOCATION are synced with this one in-place
+ *            (replaced with newly synced file)
+ *    If only target-file -t is set -> either -i in-place or -o output-file must be set
+ *    Source file can be set with -s if it should be something else than DEFAULT_SOURCE_FILE_LOCATION
+ *
+ *    If any of the paths to files/dirs given by user are not valid, an error message is printed and script gets aborted
+ */
+function parseCliInput() {
+  program
+    .option('-d, --output-dir <output-dir>', 'output dir when running script on all language files; mutually exclusive with -o')
+    .option('-t, --target-file <target>', 'target file we compare with and where completed output ends up if -o is not configured and -i is')
+    .option('-i, --edit-in-place', 'edit-in-place; store output straight in target file; mutually exclusive with -o')
+    .option('-s, --source-file <source>', 'source file to be parsed for translation', projectRoot(DEFAULT_SOURCE_FILE_LOCATION))
+    .option('-o, --output-file <output>', 'where output of script ends up; mutually exclusive with -i')
+    .usage('([-d <output-dir>] [-s <source-file>]) || (-t <target-file> (-i | -o <output>) [-s <source-file>])')
+    .parse(process.argv);
+
+  if (!program.targetFile) {
+    fs.readdirSync(projectRoot(LANGUAGE_FILES_LOCATION)).forEach(file => {
+      if (!program.sourceFile.toString().endsWith(file)) {
+        const targetFileLocation = projectRoot(LANGUAGE_FILES_LOCATION + "/" + file);
+        console.log('Syncing file at: ' + targetFileLocation + ' with source file at: ' + program.sourceFile);
+        if (program.outputDir) {
+          if (!fs.existsSync(program.outputDir)) {
+            fs.mkdirSync(program.outputDir);
+          }
+          const outputFileLocation = program.outputDir + "/" + file;
+          console.log('Output location: ' + outputFileLocation);
+          syncFileWithSource(targetFileLocation, outputFileLocation);
+        } else {
+          console.log('Replacing in target location');
+          syncFileWithSource(targetFileLocation, targetFileLocation);
+        }
+      }
+    });
+  } else {
+    if (program.targetFile && !checkIfPathToFileIsValid(program.targetFile)) {
+      console.error('Directory path of target file is not valid.');
+      console.log(program.outputHelp());
+      process.exit(1);
+    }
+    if (program.targetFile && checkIfFileExists(program.targetFile) && !(program.editInPlace || program.outputFile)) {
+      console.error('This target file already exists, if you want to overwrite this add option -i, or add an -o output location');
+      console.log(program.outputHelp());
+      process.exit(1);
+    }
+    if (!checkIfFileExists(program.sourceFile)) {
+      console.error('Path of source file is not valid.');
+      console.log(program.outputHelp());
+      process.exit(1);
+    }
+    if (program.outputFile && !checkIfPathToFileIsValid(program.outputFile)) {
+      console.error('Directory path of output file is not valid.');
+      console.log(program.outputHelp());
+      process.exit(1);
+    }
+
+    syncFileWithSource(program.targetFile, getOutputFileLocationIfExistsElseTargetFileLocation(program.targetFile));
+  }
+}
+
+/**
+ * Creates chunk lists for both the source and the target files (for example en.json5 and nl.json5 respectively)
+ *    > Creates output chunks by comparing the source chunk with corresponding target chunk (based on key of translation)
+ *    > Writes the output chunks to a new valid lang.json5 file, either replacing the target file (-i in-place)
+ *          or sending it to an output file specified by the user
+ * @param pathToTargetFile    Valid path to target file to generate target chunks from
+ * @param pathToOutputFile    Valid path to output file to write output chunks to
+ */
+function syncFileWithSource(pathToTargetFile, pathToOutputFile) {
+  const progressBar = new _cliProgress.SingleBar({}, _cliProgress.Presets.shades_classic);
+  progressBar.start(100, 0);
+
+  const sourceLines = [];
+  const targetLines = [];
+  const existingTargetFile = readFileIfExists(pathToTargetFile);
+  existingTargetFile.toString().split("\n").forEach((function (line) {
+    targetLines.push(line.trim());
+  }));
+  progressBar.update(10);
+  const sourceFile = readFileIfExists(program.sourceFile);
+  sourceFile.toString().split("\n").forEach((function (line) {
+    sourceLines.push(line.trim());
+  }));
+  progressBar.update(20);
+  const sourceChunks = createChunks(sourceLines, progressBar, false);
+  const targetChunks = createChunks(targetLines, progressBar, true);
+
+  const outputChunks = compareChunksAndCreateOutput(sourceChunks, targetChunks, progressBar);
+
+  const file = fs.createWriteStream(pathToOutputFile);
+  file.on('error', function (err) {
+    console.error('Something went wrong writing to output file at: ' + pathToOutputFile + err)
+  });
+  file.on('open', function() {
+    file.write("{\n");
+    outputChunks.forEach(function (chunk) {
+      progressBar.increment();
+      chunk.split("\n").forEach(function (line) {
+        file.write("  " + line + "\n");
+      });
+    });
+    file.write("\n}");
+    file.end();
+  });
+  file.on('finish', function() {
+    const osName = process.platform;
+    if (osName.startsWith("win")) {
+      replaceLineEndingsToCRLF(pathToOutputFile);
+    }
+  });
+
+  progressBar.update(100);
+  progressBar.stop();
+}
+
+/**
+ * For each of the source chunks:
+ *      - Determine if it's a new key-value => Add it to output, with source comments, source key-value commented, a message indicating it's new and the source-key value uncommented
+ *      - If it's not new, compare it with the corresponding target chunk and log the differences, see createNewChunkComparingSourceAndTarget
+ * @param sourceChunks      All the source chunks, split per key-value pair group
+ * @param targetChunks      All the target chunks, split per key-value pair group
+ * @param progressBar       The progressbar for the CLI
+ * @return {Array}          All the output chunks, split per key-value pair group
+ */
+function compareChunksAndCreateOutput(sourceChunks, targetChunks, progressBar) {
+  const outputChunks = [];
+  sourceChunks.map((sourceChunk) => {
+    progressBar.increment();
+    if (sourceChunk.trim().length !== 0) {
+      let newChunk = [];
+      const sourceList = sourceChunk.split("\n");
+      const keyValueSource = sourceList[sourceList.length - 1];
+      const keySource = getSubStringBeforeLastString(keyValueSource, ":");
+      const commentSource = getSubStringBeforeLastString(sourceChunk, keyValueSource);
+
+      const correspondingTargetChunk = targetChunks.find((targetChunk) => {
+        return targetChunk.includes(keySource);
+      });
+
+      // Create new chunk with: the source comments, the commented source key-value, the todos and either the old target key-value pair or if it's a new pair, the source key-value pair
+      newChunk.push(removeWhiteLines(commentSource));
+      newChunk.push("// " + keyValueSource);
+      if (correspondingTargetChunk === undefined) {
+        newChunk.push(NEW_MESSAGE_TODO);
+        newChunk.push(keyValueSource);
+      } else {
+        createNewChunkComparingSourceAndTarget(correspondingTargetChunk, sourceChunk, commentSource, keyValueSource, newChunk);
+      }
+
+      outputChunks.push(newChunk.filter(Boolean).join("\n"));
+    } else {
+      outputChunks.push(sourceChunk);
+    }
+  });
+  return outputChunks;
+}
+
+/**
+ * If a corresponding target chunk is found:
+ *      - If old key value is not found in comments > Assumed it is new key
+ *      - If the target comments do not contain the source comments (because they have changed since last time) => Add comments changed message
+ *      - If the key-value in the target comments is not the same as the source key-value (because it changes since last time) => Add message changed message
+ *      - Add the old todos if they haven't been added already
+ *      - End with the original target key-value
+ */
+function createNewChunkComparingSourceAndTarget(correspondingTargetChunk, sourceChunk, commentSource, keyValueSource, newChunk) {
+  let commentsOfSourceHaveChanged = false;
+  let messageOfSourceHasChanged = false;
+
+  const targetList = correspondingTargetChunk.split("\n");
+  const oldKeyValueInTargetComments = getSubStringWithRegex(correspondingTargetChunk, "\\s*\\/\\/\\s*\".*");
+  const keyValueTarget = targetList[targetList.length - 1];
+
+  if (oldKeyValueInTargetComments != null) {
+    const oldKeyValueUncommented = getSubStringWithRegex(oldKeyValueInTargetComments[0], "\".*")[0];
+
+    if (!(_.isEmpty(correspondingTargetChunk) && _.isEmpty(commentSource)) && !removeWhiteLines(correspondingTargetChunk).includes(removeWhiteLines(commentSource.trim()))) {
+      commentsOfSourceHaveChanged = true;
+      newChunk.push(COMMENTS_CHANGED_TODO);
+    }
+    const parsedOldKey = JSON5.stringify("{" + oldKeyValueUncommented + "}");
+    const parsedSourceKey = JSON5.stringify("{" + keyValueSource + "}");
+    if (!_.isEqual(parsedOldKey, parsedSourceKey)) {
+      messageOfSourceHasChanged = true;
+      newChunk.push(MESSAGE_CHANGED_TODO);
+    }
+    addOldTodosIfNeeded(targetList, newChunk, commentsOfSourceHaveChanged, messageOfSourceHasChanged);
+  }
+  newChunk.push(keyValueTarget);
+}
+
+// Adds old todos found in target comments if they've not been added already
+function addOldTodosIfNeeded(targetList, newChunk, commentsOfSourceHaveChanged, messageOfSourceHasChanged) {
+  targetList.map((targetLine) => {
+    const foundTODO = getSubStringWithRegex(targetLine, "\\s*//\\s*TODO.*");
+    if (foundTODO != null) {
+      const todo = foundTODO[0];
+      if (!((todo.includes(COMMENTS_CHANGED_TODO) && commentsOfSourceHaveChanged)
+        || (todo.includes(MESSAGE_CHANGED_TODO) && messageOfSourceHasChanged))) {
+        newChunk.push(todo);
+      }
+    }
+  });
+}
+
+/**
+ * Creates chunks from an array of lines, each chunk contains either an empty line or a grouping of comments with their corresponding key-value pair
+ * @param lines             Array of lines, to be grouped into chunks
+ * @param progressBar       Progressbar of the CLI
+ * @return {Array}          Array of chunks, grouped by key-value and their corresponding comments or an empty line
+ */
+function createChunks(lines, progressBar, creatingTarget) {
+  const chunks = [];
+  let nextChunk = [];
+  let onMultiLineComment = false;
+  lines.map((line) => {
+    progressBar.increment();
+    if (line.length === 0) {
+      chunks.push(line);
+    }
+    if (isOneLineCommentLine(line)) {
+      nextChunk.push(line);
+    }
+    if (onMultiLineComment) {
+      nextChunk.push(line);
+      if (isEndOfMultiLineComment(line)) {
+        onMultiLineComment = false;
+      }
+    }
+    if (isStartOfMultiLineComment(line)) {
+      nextChunk.push(line);
+      onMultiLineComment = true;
+    }
+    if (isKeyValuePair(line)) {
+      nextChunk.push(line);
+      const newMessageLineIfExists = nextChunk.find((lineInChunk) => lineInChunk.trim().startsWith(NEW_MESSAGE_TODO));
+      if (newMessageLineIfExists === undefined || !creatingTarget) {
+        chunks.push(nextChunk.join("\n"));
+      }
+      nextChunk = [];
+    }
+  });
+  return chunks;
+}
+
+function readFileIfExists(pathToFile) {
+  if (checkIfFileExists(pathToFile)) {
+    try {
+      return fs.readFileSync(pathToFile, 'utf8');
+    } catch (e) {
+      console.error('Error:', e.stack);
+    }
+  }
+  return null;
+}
+
+function isOneLineCommentLine(line) {
+  return (line.startsWith("//"));
+}
+
+function isStartOfMultiLineComment(line) {
+  return (line.startsWith("/*"));
+}
+
+function isEndOfMultiLineComment(line) {
+  return (line.endsWith("*/"));
+}
+
+function isKeyValuePair(line) {
+  return (line.startsWith("\""));
+}
+
+
+function getSubStringWithRegex(string, regex) {
+  return string.match(regex);
+}
+
+function getSubStringBeforeLastString(string, char) {
+  const lastCharIndex = string.lastIndexOf(char);
+  return string.substr(0, lastCharIndex);
+}
+
+
+function getOutputFileLocationIfExistsElseTargetFileLocation(targetLocation) {
+  if (program.outputFile) {
+    return program.outputFile;
+  }
+  return targetLocation;
+}
+
+function checkIfPathToFileIsValid(pathToCheck) {
+  if (!pathToCheck.includes("/")) {
+    return true;
+  }
+  return checkIfFileExists(getPathOfDirectory(pathToCheck));
+}
+
+function checkIfFileExists(pathToCheck) {
+  return fs.existsSync(pathToCheck);
+}
+
+function getPathOfDirectory(pathToCheck) {
+  return getSubStringBeforeLastString(pathToCheck, "/");
+}
+
+function removeWhiteLines(string) {
+  return string.replace(/^(?=\n)$|^\s*|\s*$|\n\n+/gm, "")
+}
+
+/**
+ * Replaces UNIX \n LF line endings to windows \r\n CRLF line endings.
+ * @param filePath  Path to file whose line endings are being converted
+ */
+function replaceLineEndingsToCRLF(filePath) {
+  const data = readFileIfExists(filePath);
+  const result = data.replace(/\n/g,"\r\n");
+  fs.writeFileSync(filePath, result, 'utf8');
+}

yarn.lock

@@ -2053,6 +2053,14 @@ cli-cursor@^2.1.0:
   dependencies:
     restore-cursor "^2.0.0"
 
+cli-progress@^3.3.1:
+  version "3.3.1"
+  resolved "https://registry.yarnpkg.com/cli-progress/-/cli-progress-3.3.1.tgz#e08fb6853e269c3bc7e0ad8ad0ede59035f5fbce"
+  integrity sha512-cfvv/uuWblzSI6fvpmCNREWxwzI/luelp+P7zoPlguswswWVCfsq334/U6tmh+vusJ7yzbQ3xFni9JNqiF4fAA==
+  dependencies:
+    colors "^1.1.2"
+    string-width "^2.1.1"
+
 cli-width@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639"
@@ -2224,6 +2232,11 @@ colors@^1.1.0:
   resolved "https://registry.yarnpkg.com/colors/-/colors-1.3.2.tgz#2df8ff573dfbf255af562f8ce7181d6b971a359b"
   integrity sha512-rhP0JSBGYvpcNQj4s5AdShMeE5ahMop96cTeDl/v9qQQm2fYClE2QXZRi8wLzc+GmXSxdIqqbOIAhyObEXDbfQ==
 
+colors@^1.1.2:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
+  integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
+
 combine-lists@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/combine-lists/-/combine-lists-1.0.1.tgz#458c07e09e0d900fc28b70a3fec2dacd1d2cb7f6"
@@ -2253,6 +2266,11 @@ commander@^2.12.1, commander@^2.18.0, commander@~2.20.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
   integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==
 
+commander@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-3.0.2.tgz#6837c3fb677ad9933d1cfba42dd14d5117d6b39e"
+  integrity sha512-Gar0ASD4BDyKC4hl4DwHqDrmvjoxWKZigVnAbn5H1owvm4CxCPdb0HQDehwNYMJpla5+M2tPmPARzhtYuwpHow==
+
 commander@~2.13.0:
   version "2.13.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.13.0.tgz#6964bca67685df7c1f1430c584f07d7597885b9c"