import path from 'node:path'; import crypto from 'node:crypto'; import { log, walk, DEFAULT_LANGUAGE, performApiRequest, RESOURCES_ROOT, DOCUMENTS_EXT, ORIGINAL_MKDOCS_CONFIG, CROWIN_MKDOCS_CONFIG, MKDOCS_CONTENT_TYPE, } from './crowdin-common.mjs'; import {fs} from '@appium/support'; const LANGUAGE_ROOT = path.resolve(RESOURCES_ROOT, DEFAULT_LANGUAGE); // Max supported value is 500 const MAX_ITEMS_PER_PAGE = 300; const DOCUMENT_CONTENT_TYPE = 'text/markdown'; /** * * @param {string} str * @returns {string} */ function toHash(str) { return crypto.createHash('md5').update(str).digest('hex'); } /** * * @param {string} fullPath * @returns {string} */ function toCrowdinPath(fullPath) { const fileName = path.basename(fullPath); const isMkDocsConfig = fileName === ORIGINAL_MKDOCS_CONFIG(DEFAULT_LANGUAGE); let result = `/${path.relative(LANGUAGE_ROOT, fullPath)}`; if (isMkDocsConfig || result.includes('..')) { return `/${isMkDocsConfig ? CROWIN_MKDOCS_CONFIG : fileName}`; } return result; } /** * * @param {string} name * @param {string|null|undefined} [parentId] * @returns {Promise} */ async function addDirectory(name, parentId) { const {data: directoryData} = await performApiRequest('/directories', { method: 'POST', payload: { name, directoryId: parentId, } }); return directoryData.id; } /** * * @param {string} name * @param {number} storageId * @param {string|null|undefined} [parentDirectoryId] * @returns {Promise} */ async function addFile(name, storageId, parentDirectoryId) { const {data: fileData} = await performApiRequest('/files', { method: 'POST', payload: { name, storageId, directoryId: parentDirectoryId, } }); return fileData.id; } /** * @returns {Promise[]>} */ async function listFiles() { const {data: filesData} = await performApiRequest('/files', { method: 'GET', params: { limit: MAX_ITEMS_PER_PAGE, } }); return filesData.map(({data}) => data); } /** * * @param {number} fileId * @returns {Promise} */ async function deleteFile(fileId) { await performApiRequest(`/files/${fileId}`, { method: 'DELETE', }); } /** * * @param {number} [parentDirectoryId] * @param {number} [recursion] * @return {Promise[]>} */ async function listDirectories(parentDirectoryId) { const {data: directoriesData} = await performApiRequest('/directories', { method: 'GET', params: { limit: MAX_ITEMS_PER_PAGE, directoryId: parentDirectoryId, recursion: parentDirectoryId ? 10 : undefined, }, }); return directoriesData.map(({data}) => data); } /** * * @param {string} name Should be properly url-encoded * @param {string} fullPath * @param {string} contentType Should be one of https://www.iana.org/assignments/media-types/media-types.xhtml * @returns {Promise>} */ async function addStorage(name, fullPath, contentType) { const {data: storageData} = await performApiRequest('/storages', { method: 'POST', headers: { 'Crowdin-API-FileName': name, 'Content-Type': contentType, }, payload: fs.createReadStream(fullPath), isProjectSpecific: false, }); return storageData; } /** * * @param {string[]} matchedFiles * @returns {Promise>} */ async function uploadDocumentsToStorage(matchedFiles) { const resultMap = {}; let count = 0; for (const matchedFilePath of matchedFiles) { const crowdinPath = toCrowdinPath(matchedFilePath); // Hashing is used to make sure we always create the same storage for the same file path in Crowdin const storageName = toHash(crowdinPath); log.info(`Uploading '${crowdinPath}' to Crowdin storage (${++count} of ${matchedFiles.length})`); const storageData = await addStorage(storageName, matchedFilePath, DOCUMENT_CONTENT_TYPE); resultMap[matchedFilePath] = storageData.id; } return resultMap; } /** * * @param {string[]} matchedFiles * @returns {Promise>} */ async function ensureDirectoryStructure(matchedFiles) { const uniqueFolderPaths = new Set(); for (const matchedFile of matchedFiles) { const relativePath = path.relative(LANGUAGE_ROOT, path.dirname(matchedFile)); if (relativePath) { uniqueFolderPaths.add(relativePath); } } if (uniqueFolderPaths.size === 0) { return {}; } const splitPaths = Array.from(uniqueFolderPaths) .map((p) => p.split(path.sep)) .sort((a, b) => a.length - b.length); const result = {}; const topDirectories = await listDirectories(); for (const splitPath of splitPaths) { for (const level in splitPath) { const pathInCrowdin = `/${splitPath.slice(0, level + 1).join('/')}`; const parentPathInCrowdin = path.dirname(pathInCrowdin); const parentDirectoryId = level === 0 ? undefined : result[parentPathInCrowdin]; const subDirectories = level === 0 ? topDirectories : await listDirectories(parentDirectoryId); const existingDirectoryData = subDirectories.find((data) => data.path === pathInCrowdin); if (existingDirectoryData) { log.info(`Crowdin directory '${pathInCrowdin}' already exists`); result[pathInCrowdin] = existingDirectoryData.id; } else { log.info(`Crowdin directory '${pathInCrowdin}' does not exist yet. Adding it`); result[pathInCrowdin] = await addDirectory(splitPath[level], parentDirectoryId); } } } return result; } /** * * @param {Record} storageMapping * @param {Record} directoriesMapping * @param {Record[]} existingFilesData * @returns {Promise>} */ async function ensureFileStructure(storageMapping, directoriesMapping, existingFilesData) { const result = {}; let count = 0; for (const [fullPath, storageId] of Object.entries(storageMapping)) { const pathInCrowdin = toCrowdinPath(fullPath); log.info(`Synchronizing '${pathInCrowdin}' (${++count} of ${Object.keys(storageMapping).length})`); const fileData = existingFilesData.find((data) => data.path === pathInCrowdin); if (fileData) { result[fullPath] = fileData.id; } else { const parentFolderId = directoriesMapping[path.dirname(pathInCrowdin)]; try { const fileId = await addFile(encodeURIComponent(path.basename(pathInCrowdin)), storageId, parentFolderId); result[fullPath] = fileId; } catch (e) { log.warn(`Cannot add '${pathInCrowdin}'. Skipping it`); if (e.response) { log.info(`Crowdin API status: ${e.response.status}`); log.info(`Crowdin API error: ${JSON.stringify(e.response.data, null, 2)}`); } else { log.info(e); } continue; } } } return result; } /** * @returns {Promise} */ async function cleanupObsoleteDocuments() { const [existingFilesData, matchedFiles] = await Promise.all([ listFiles(), walk(LANGUAGE_ROOT, DOCUMENTS_EXT) ]); const matchedFilePaths = new Set(matchedFiles.map(toCrowdinPath)); let count = 0; for (const existingFileData of existingFilesData) { if (matchedFilePaths.has(existingFileData.path) || !existingFileData.name.endsWith(DOCUMENTS_EXT)) { continue; } log.info(`Deleting the obsolete document '${existingFileData.path}'`); await deleteFile(existingFileData.id); ++count; } if (count > 0) { log.info(`Deleted ${count} obsolete documents`); } } /** * * @param {Record} filesMapping * @param {Record} storageMapping * @returns {Promise} */ async function updateFiles(filesMapping, storageMapping) { let count = 0; for (const [fullPath, fileId] of Object.entries(filesMapping)) { log.info(`Updating '${toCrowdinPath(fullPath)}' (${++count} of ${Object.keys(filesMapping).length})`); await performApiRequest(`/files/${fileId}`, { method: 'PUT', payload: { storageId: storageMapping[fullPath], }, }); } } /** * * @returns {Promise} */ async function updateDocuments() { const matchedFiles = await walk(LANGUAGE_ROOT, DOCUMENTS_EXT); if (matchedFiles.length === 0) { throw new Error(`Did not find any files matching the '*${DOCUMENTS_EXT}' extension in '${LANGUAGE_ROOT}'`); } log.info(`Matched ${matchedFiles.length} files from '${LANGUAGE_ROOT}' for upload...`); const [storageMapping, directoriesMapping] = await Promise.all([ uploadDocumentsToStorage(matchedFiles), ensureDirectoryStructure(matchedFiles), ]); const existingFilesData = await listFiles(); const filesMapping = await ensureFileStructure(storageMapping, directoriesMapping, existingFilesData); await updateFiles(filesMapping, storageMapping); } /** * @returns {Promise} */ async function updateMkDocsConfig() { const configFileName = ORIGINAL_MKDOCS_CONFIG(DEFAULT_LANGUAGE); const matchedFilePath = path.join(RESOURCES_ROOT, configFileName); if (!await fs.exists(matchedFilePath)) { throw new Error(`Did not find the MkDocs config at '${matchedFilePath}'`); } const storageData = await addStorage(encodeURIComponent(CROWIN_MKDOCS_CONFIG), matchedFilePath, MKDOCS_CONTENT_TYPE); const storageMapping = {[matchedFilePath]: storageData.id}; const existingFilesData = await listFiles(); const filesMapping = await ensureFileStructure(storageMapping, {}, existingFilesData); await updateFiles(filesMapping, storageMapping); } async function main() { log.info('Updating documents'); await updateDocuments(); await cleanupObsoleteDocuments(); log.info('Updating MkDocs config'); await updateMkDocsConfig(); log.info('All done'); } (async () => await main())();