# SPDX-License-Identifier: LGPL-2.1-or-later
# ***************************************************************************
# * *
# * Copyright (c) 2025 The FreeCAD project association AISBL *
# * *
# * This file is part of the FreeCAD Addon Manager. *
# * *
# * This is free software: you can redistribute it and/or modify it *
# * under the terms of the GNU Lesser General Public License as *
# * published by the Free Software Foundation, either version 2.1 of the *
# * License, or (at your option) any later version. *
# * *
# * The software is distributed in the hope that it will be useful, but *
# * WITHOUT ANY WARRANTY; without even the implied warranty of *
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
# * Lesser General Public License for more details. *
# * *
# * You should have received a copy of the GNU Lesser General Public *
# * License along with the FreeCAD Addon Manager. If not, see *
# * . *
# * *
# ***************************************************************************
"""Run a full CrowdIn translation cycle for the Addon Manager. Requires that the
CrowdIn API token is stored in ~/.crowdin-freecad-token, and that lupdate be in PATH."""
import collections
import datetime
import json
import os
import shutil
import stat
import subprocess
import sys
import tempfile
import time
from functools import lru_cache
from urllib.parse import quote_plus
from urllib.request import Request, urlopen, urlretrieve
CROWDIN_API_URL = "https://api.crowdin.com/api/v2"
CROWDIN_API_PROJECT_ID = "freecad-addons"
CROWDIN_PROJECT_NAME = "AddonManager"
CROWDIN_FILE_NAME = "AddonManager.ts"
TS_FILE_PATH = os.curdir
BASE_FILENAME = "AddonManager"
MIN_TRANSLATION_THRESHOLD = 0.5
# The CrowdinUpdater class is from FreeCAD's updatecrowdin.py script
# SPDX-License-Identifier: LGPL-2.1-or-later
# Copyright (c) 2015 Yorik van Havre
# Copyright (c) 2021 Benjamin Nauck
# Copyright (c) 2021 Mattias Pierre
class CrowdinUpdater:
BASE_URL = CROWDIN_API_URL
def __init__(self, token, project_identifier):
self.token = token
self.project_identifier = project_identifier
self.multithread = False
@lru_cache()
def _get_project_id(self):
url = f"{self.BASE_URL}/projects/"
response = self._make_api_req(url)
for project in [p["data"] for p in response]:
if project["identifier"] == self.project_identifier:
return project["id"]
raise Exception("No project identifier found!")
def _make_project_api_req(self, project_path, *args, **kwargs):
url = f"{self.BASE_URL}/projects/{self._get_project_id()}{project_path}"
return self._make_api_req(url=url, *args, **kwargs)
def _make_api_req(self, url, extra_headers=None, method="GET", data=None):
if extra_headers is None:
extra_headers = {}
headers = {"Authorization": "Bearer " + load_token(), **extra_headers}
if type(data) is dict:
headers["Content-Type"] = "application/json"
data = json.dumps(data).encode("utf-8")
request = Request(url, headers=headers, method=method, data=data)
request_result = urlopen(request)
if request_result.getcode() >= 300:
print(f"Failed to make API request {url}: return code {request_result.getcode()}")
raise Exception("Failed to make API request")
return json.loads(request_result.read())["data"]
def _get_files_info(self):
files = self._make_project_api_req("/files?limit=250")
return {f["data"]["path"].strip("/"): str(f["data"]["id"]) for f in files}
def _add_storage(self, filename, fp):
response = self._make_api_req(
f"{self.BASE_URL}/storages",
data=fp,
method="POST",
extra_headers={
"Crowdin-API-FileName": filename,
"Content-Type": "application/octet-stream",
},
)
return response["id"]
def _update_file(self, ts_file, files_info):
filename = quote_plus(ts_file)
with open(os.path.join(TS_FILE_PATH, ts_file), "rb") as fp:
storage_id = self._add_storage(filename, fp)
if filename in files_info:
file_id = files_info[filename]
self._make_project_api_req(
f"/files/{file_id}",
method="PUT",
data={
"storageId": storage_id,
"updateOption": "keep_translations_and_approvals",
},
)
print(f"{filename} updated")
else:
self._make_project_api_req("/files", data={"storageId": storage_id, "name": filename})
print(f"{filename} uploaded")
def status(self):
response = self._make_project_api_req("/languages/progress?limit=100")
return [item["data"] for item in response]
def download(self, build_id):
filename = f"{self.project_identifier}.zip"
response = self._make_project_api_req(f"/translations/builds/{build_id}/download")
urlretrieve(response["url"], filename)
print("download of " + filename + " complete")
def build(self):
self._make_project_api_req("/translations/builds", data={}, method="POST")
def build_status(self):
response = self._make_project_api_req("/translations/builds")
return [item["data"] for item in response]
def wait_for_build_completion(self):
while True:
status = self.build_status()
still_running = False
for builds in status:
if builds["status"] == "inProgress":
still_running = True
if not still_running:
print("done.")
return
print(".", end="")
time.sleep(10)
def update(self, ts_files):
files_info = self._get_files_info()
for ts_file in ts_files:
self._update_file(ts_file, files_info)
def load_token():
"""returns the CrowdIn API token, either from the CROWDIN_API_TOKEN environment variable or
from a file in the user's home directory. If neither is found, returns None."""
if os.environ.get("CROWDIN_API_TOKEN"):
return os.environ.get("CROWDIN_API_TOKEN")
config_file = os.path.expanduser("~") + os.sep + ".crowdin-freecad-token"
if os.path.exists(config_file):
with open(config_file) as file:
return file.read().strip()
return None
def process_single_translation_file(source_path: str, target_path: str):
"""updates a single ts file and creates a corresponding qm file"""
basename = os.path.basename(source_path)
new_path = os.path.join(target_path, basename)
shutil.copyfile(source_path, new_path)
print("Generating qm file for", basename, "...")
try:
subprocess.run(
[
"lrelease",
new_path,
],
timeout=5,
)
except Exception as e:
print(e)
new_qm = new_path[:-3] + ".qm"
if not os.path.exists(new_qm):
print("ERROR: failed to create " + new_qm + ", aborting")
sys.exit()
temp_folder = tempfile.mkdtemp()
def apply_translations_for_language(language_file: str):
"""treats a single language"""
print(f"Processing {language_file}...")
source_path = os.path.join(temp_folder, CROWDIN_PROJECT_NAME, language_file)
target_path = os.path.abspath(TS_FILE_PATH)
process_single_translation_file(source_path, target_path)
def get_language_percentage(language_file: str):
"""returns the percentage of translated strings for a language -- this operates on the local
file with no API interaction."""
if not os.path.exists(language_file):
return 0
source_counter = 0
unfinished_counter = 0
with open(language_file, "r", encoding="utf-8") as f:
for line in f:
if "" in line:
source_counter = source_counter + 1
elif 'type="unfinished"' in line:
unfinished_counter = unfinished_counter + 1
if source_counter > 0:
return (source_counter - unfinished_counter) / source_counter
return 0
def rename_locale_to_two_letter_code():
"""Find all the files where it is possible to rename unambiguously and drop the second part of
the locale string, leaving only the two-letter code."""
base_path = os.path.join(temp_folder, CROWDIN_PROJECT_NAME)
ts_files = sorted(os.listdir(base_path))
# All files are named AddonManager_.ts, where locale consists of a 2- or 3-letter
# language code, followed by a dash, followed by a two-letter country code. Don't bother with
# a regex, just figure out the lengths of the parts we're interested in.
prefix_len = len(CROWDIN_PROJECT_NAME) + 1
suffix_len = len("-xx.ts")
codes_only = [f[prefix_len:-suffix_len] for f in ts_files]
codes_that_need_disambiguation = []
locale_frequency = collections.Counter(codes_only)
for code in codes_only:
if code not in codes_that_need_disambiguation and locale_frequency[code] > 1:
codes_that_need_disambiguation.append(code)
for ts_file in ts_files:
code = ts_file[prefix_len:-suffix_len]
if code not in codes_that_need_disambiguation:
new_name = CROWDIN_PROJECT_NAME + "_" + code + ".ts"
os.rename(os.path.join(base_path, ts_file), os.path.join(base_path, new_name))
def apply_all_available_translations():
"""treats all languages"""
base_path = os.path.join(temp_folder, CROWDIN_PROJECT_NAME)
for language_file in os.listdir(base_path):
percentage = get_language_percentage(os.path.join(base_path, language_file))
if percentage >= MIN_TRANSLATION_THRESHOLD:
apply_translations_for_language(language_file)
else:
print(
"Skipping {} because it is not translated enough ({} %)".format(
language_file, round(100 * percentage, 0)
)
)
def run_and_download_build(crowdin_updater: CrowdinUpdater):
"""runs a build (if needed) and downloads the latest translations"""
# First, determine when the last build was created
build_status = crowdin_updater.build_status()
last_build_id = None
last_build_date = None
for build in build_status:
if build["status"] == "finished":
build_id = build["id"]
build_date = datetime.datetime.fromisoformat(build["finishedAt"])
if last_build_id is None or last_build_date is None or build_date > last_build_date:
last_build_id = build_id
last_build_date = build_date
# If the last build was not in the last hour, build a new one
if last_build_date is None or datetime.datetime.now(
tz=datetime.timezone.utc
) - last_build_date > datetime.timedelta(hours=1):
print("Last build was not in the last hour: running a new build", end="")
crowdin_updater.build()
crowdin_updater.wait_for_build_completion()
print("Build complete, waiting ten seconds for translations to be ready...")
time.sleep(10)
# Download the latest translations
print(f"Downloading latest translations (build ID {last_build_id})...")
crowdin_updater.download(last_build_id)
if __name__ == "__main__":
token = load_token()
if not token:
print("ERROR: no API token found, aborting")
sys.exit(1)
crowdin = CrowdinUpdater(token, CROWDIN_API_PROJECT_ID)
# The first half of the cycle: download and apply existing translations:
run_and_download_build(crowdin)
shutil.unpack_archive(f"{CROWDIN_API_PROJECT_ID}.zip", temp_folder)
rename_locale_to_two_letter_code()
apply_all_available_translations()
# The other side of the cycle: gather the new strings and send to CrowdIn:
files_to_translate = []
skip_dirs = ["__pycache__", "CatalogCache", "AddonManagerTest"]
found = False
toplevel_path = os.path.abspath(TS_FILE_PATH)
while not found:
# Find the top-level AddonManager directory, which contains the AddonManager.py file
if os.path.exists(os.path.join(toplevel_path, "AddonManager.py")):
found = True
else:
toplevel_path = os.path.abspath(os.path.join(toplevel_path, ".."))
for root, dirs, files in os.walk(toplevel_path):
dirs[:] = [d for d in dirs if not d.startswith(".") and d not in skip_dirs]
files = [f for f in files if not f.startswith(".")]
for file in files:
if file.endswith(".py") or file.endswith(".ui"):
files_to_translate.append(os.path.abspath(os.path.join(root, file)))
list_file = os.path.join(temp_folder, "files_to_extract.txt")
with open(list_file, "w", encoding="utf-8") as f:
for file in files_to_translate:
f.write(file + "\n")
print("Running lupdate to generate new translation files...")
args = [
"lupdate",
"-no-obsolete",
"-no-ui-lines",
"-locations",
"relative",
f"@{list_file}",
"-ts",
os.path.join(TS_FILE_PATH, CROWDIN_FILE_NAME),
]
result = subprocess.run(
args,
timeout=30,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
print(result.stdout.decode("utf-8"))
print("Sending new translation file to CrowdIn...")
crowdin.update([CROWDIN_FILE_NAME])
def try_harder(func, path, _exc_info):
try:
os.chmod(path, stat.S_IWRITE)
func(path)
except (OSError, PermissionError, FileNotFoundError):
pass # Well, we tried! The OS will have to clean it up.
shutil.rmtree(temp_folder, onerror=try_harder)
try:
os.remove(f"{CROWDIN_API_PROJECT_ID}.zip")
except (OSError, FileNotFoundError):
pass # Couldn't delete it, might already be gone... nothing to do here.