devsite/node_modules/@11ty/eleventy/src/TemplateData.js
2024-07-07 18:49:38 -07:00

672 lines
21 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

const fs = require("fs");
const path = require("path");
const { set: lodashset, get: lodashget } = require("@11ty/lodash-custom");
const { TemplatePath, isPlainObject } = require("@11ty/eleventy-utils");
const merge = require("./Util/Merge");
const unique = require("./Util/Unique");
const TemplateGlob = require("./TemplateGlob");
const EleventyExtensionMap = require("./EleventyExtensionMap");
const EleventyBaseError = require("./EleventyBaseError");
const TemplateDataInitialGlobalData = require("./TemplateDataInitialGlobalData");
const { EleventyRequire } = require("./Util/Require");
const debugWarn = require("debug")("Eleventy:Warnings");
const debug = require("debug")("Eleventy:TemplateData");
const debugDev = require("debug")("Dev:Eleventy:TemplateData");
class FSExistsCache {
constructor() {
this._cache = new Map();
}
has(path) {
return this._cache.has(path);
}
exists(path) {
let exists = this._cache.get(path);
if (!this.has(path)) {
exists = fs.existsSync(path);
this._cache.set(path, exists);
}
return exists;
}
markExists(path, value = true) {
this._cache.set(path, !!value);
}
}
class TemplateDataConfigError extends EleventyBaseError {}
class TemplateDataParseError extends EleventyBaseError {}
class TemplateData {
constructor(inputDir, eleventyConfig) {
if (!eleventyConfig) {
throw new TemplateDataConfigError("Missing `config`.");
}
this.eleventyConfig = eleventyConfig;
this.config = this.eleventyConfig.getConfig();
this.benchmarks = {
data: this.config.benchmarkManager.get("Data"),
aggregate: this.config.benchmarkManager.get("Aggregate"),
};
this.inputDirNeedsCheck = false;
this.setInputDir(inputDir);
this.rawImports = {};
this.globalData = null;
this.templateDirectoryData = {};
// It's common for data files not to exist, so we avoid going to the FS to
// re-check if they do via a quick-and-dirty cache.
this._fsExistsCache = new FSExistsCache();
this.initialGlobalData = new TemplateDataInitialGlobalData(this.eleventyConfig);
}
setFileSystemSearch(fileSystemSearch) {
this.fileSystemSearch = fileSystemSearch;
}
get extensionMap() {
if (!this._extensionMap) {
this._extensionMap = new EleventyExtensionMap([], this.eleventyConfig);
}
return this._extensionMap;
}
set extensionMap(map) {
this._extensionMap = map;
}
get environmentVariables() {
return this._env;
}
set environmentVariables(env) {
this._env = env;
}
/* Used by tests */
_setConfig(config) {
this.config = config;
}
setInputDir(inputDir) {
this.inputDirNeedsCheck = true;
this.inputDir = inputDir;
this.dataDir = this.config.dir.data
? TemplatePath.join(inputDir, this.config.dir.data)
: inputDir;
}
getRawImports() {
let pkgPath = TemplatePath.absolutePath("package.json");
try {
this.rawImports[this.config.keys.package] = require(pkgPath);
} catch (e) {
debug("Could not find and/or require package.json for data preprocessing at %o", pkgPath);
}
return this.rawImports;
}
getDataDir() {
return this.dataDir;
}
clearData() {
this.globalData = null;
this.configApiGlobalData = null;
this.templateDirectoryData = {};
}
_getGlobalDataGlobByExtension(dir, extension) {
return TemplateGlob.normalizePath(
dir,
"/",
this.config.dir.data !== "." ? this.config.dir.data : "",
`/**/*.${extension}`
);
}
async _checkInputDir() {
if (this.inputDirNeedsCheck) {
let globalPathStat = await fs.promises.stat(this.inputDir);
if (!globalPathStat.isDirectory()) {
throw new Error("Could not find data path directory: " + this.inputDir);
}
this.inputDirNeedsCheck = false;
}
}
async getInputDir() {
let dir = ".";
if (this.inputDir) {
await this._checkInputDir();
dir = this.inputDir;
}
return dir;
}
// This is a backwards compatibility helper with the old `jsDataFileSuffix` configuration API
getDataFileSuffixes() {
// New API
if (Array.isArray(this.config.dataFileSuffixes)) {
return this.config.dataFileSuffixes;
}
// Backwards compatibility
if (this.config.jsDataFileSuffix) {
let suffixes = [];
suffixes.push(this.config.jsDataFileSuffix); // e.g. filename.11tydata.json
suffixes.push(""); // suffix-less for free with old API, e.g. filename.json
return suffixes;
}
return []; // if both of these entries are set to false, use no files
}
// This is used exclusively for --watch and --serve chokidar targets
async getTemplateDataFileGlob() {
let suffixes = this.getDataFileSuffixes();
let globSuffixesWithLeadingDot = new Set();
globSuffixesWithLeadingDot.add("json"); // covers .11tydata.json too
let globSuffixesWithoutLeadingDot = new Set();
// Typically using [ '.11tydata', '' ] suffixes to find data files
for (let suffix of suffixes) {
// TODO the `suffix` truthiness check is purely for backwards compat?
if (suffix && typeof suffix === "string") {
if (suffix.startsWith(".")) {
// .suffix.js
globSuffixesWithLeadingDot.add(`${suffix.slice(1)}.cjs`);
globSuffixesWithLeadingDot.add(`${suffix.slice(1)}.js`);
} else {
// "suffix.js" without leading dot
globSuffixesWithoutLeadingDot.add(`${suffix || ""}.cjs`);
globSuffixesWithoutLeadingDot.add(`${suffix || ""}.js`);
}
}
}
// Configuration Data Extensions e.g. yaml
if (this.hasUserDataExtensions()) {
for (let extension of this.getUserDataExtensions()) {
globSuffixesWithLeadingDot.add(extension); // covers .11tydata.{extension} too
}
}
let dir = await this.getInputDir();
let paths = [];
if (globSuffixesWithLeadingDot.size > 0) {
paths.push(`${dir}/**/*.{${Array.from(globSuffixesWithLeadingDot).join(",")}}`);
}
if (globSuffixesWithoutLeadingDot.size > 0) {
paths.push(`${dir}/**/*{${Array.from(globSuffixesWithoutLeadingDot).join(",")}}`);
}
return TemplatePath.addLeadingDotSlashArray(paths);
}
// For spidering dependencies
// TODO Can we reuse getTemplateDataFileGlob instead? Maybe just filter off the .json files before scanning for dependencies
async getTemplateJavaScriptDataFileGlob() {
let dir = await this.getInputDir();
let paths = [];
let suffixes = this.getDataFileSuffixes();
for (let suffix of suffixes) {
if (suffix) {
// TODO this check is purely for backwards compat and I kinda feel like it shouldnt be here
// paths.push(`${dir}/**/*${suffix || ""}.cjs`); // Same as above
paths.push(`${dir}/**/*${suffix || ""}.js`);
}
}
return TemplatePath.addLeadingDotSlashArray(paths);
}
async getGlobalDataGlob() {
let dir = await this.getInputDir();
let extGlob = this.getGlobalDataExtensionPriorities().join(",");
return [this._getGlobalDataGlobByExtension(dir, "{" + extGlob + "}")];
}
getWatchPathCache() {
return this.pathCache;
}
getGlobalDataExtensionPriorities() {
return this.getUserDataExtensions().concat(["json", "cjs", "js"]);
}
static calculateExtensionPriority(path, priorities) {
for (let i = 0; i < priorities.length; i++) {
let ext = priorities[i];
if (path.endsWith(ext)) {
return i;
}
}
return priorities.length;
}
async getGlobalDataFiles() {
let priorities = this.getGlobalDataExtensionPriorities();
let fsBench = this.benchmarks.aggregate.get("Searching the file system (data)");
fsBench.before();
let globs = await this.getGlobalDataGlob();
let paths = await this.fileSystemSearch.search("global-data", globs);
fsBench.after();
// sort paths according to extension priorities
// here we use reverse ordering, because paths with bigger index in array will override the first ones
// example [path/file.json, path/file.js] here js will override json
paths = paths.sort((first, second) => {
let p1 = TemplateData.calculateExtensionPriority(first, priorities);
let p2 = TemplateData.calculateExtensionPriority(second, priorities);
if (p1 < p2) {
return -1;
}
if (p1 > p2) {
return 1;
}
return 0;
});
this.pathCache = paths;
return paths;
}
getObjectPathForDataFile(dataFilePath) {
let reducedPath = TemplatePath.stripLeadingSubPath(dataFilePath, this.dataDir);
let parsed = path.parse(reducedPath);
let folders = parsed.dir ? parsed.dir.split("/") : [];
folders.push(parsed.name);
return folders;
}
async getAllGlobalData() {
let rawImports = this.getRawImports();
let globalData = {};
let files = TemplatePath.addLeadingDotSlashArray(await this.getGlobalDataFiles());
this.config.events.emit("eleventy.globalDataFiles", files);
let dataFileConflicts = {};
for (let j = 0, k = files.length; j < k; j++) {
let data = await this.getDataValue(files[j], rawImports);
let objectPathTarget = this.getObjectPathForDataFile(files[j]);
// Since we're joining directory paths and an array is not useable as an objectkey since two identical arrays are not double equal,
// we can just join the array by a forbidden character ("/"" is chosen here, since it works on Linux, Mac and Windows).
// If at some point this isn't enough anymore, it would be possible to just use JSON.stringify(objectPathTarget) since that
// is guaranteed to work but is signifivcantly slower.
let objectPathTargetString = objectPathTarget.join(path.sep);
// if two global files have the same path (but different extensions)
// and conflict, lets merge them.
if (dataFileConflicts[objectPathTargetString]) {
debugWarn(
`merging global data from ${files[j]} with an already existing global data file (${dataFileConflicts[objectPathTargetString]}). Overriding existing keys.`
);
let oldData = lodashget(globalData, objectPathTarget);
data = TemplateData.mergeDeep(this.config, oldData, data);
}
dataFileConflicts[objectPathTargetString] = files[j];
debug(`Found global data file ${files[j]} and adding as: ${objectPathTarget}`);
lodashset(globalData, objectPathTarget, data);
}
return globalData;
}
async getInitialGlobalData() {
if (!this.configApiGlobalData) {
this.configApiGlobalData = new Promise(async (resolve) => {
let globalData = await this.initialGlobalData.getData();
if (this.environmentVariables) {
if (!("env" in globalData.eleventy)) {
globalData.eleventy.env = {};
}
Object.assign(globalData.eleventy.env, this.environmentVariables);
}
resolve(globalData);
});
}
return this.configApiGlobalData;
}
async getGlobalData() {
let rawImports = this.getRawImports();
if (!this.globalData) {
this.globalData = new Promise(async (resolve) => {
let configApiGlobalData = await this.getInitialGlobalData();
let globalJson = await this.getAllGlobalData();
let mergedGlobalData = merge(globalJson, configApiGlobalData);
// OK: Shallow merge when combining rawImports (pkg) with global data files
resolve(Object.assign({}, mergedGlobalData, rawImports));
});
}
return this.globalData;
}
/* Template and Directory data files */
async combineLocalData(localDataPaths) {
let localData = {};
if (!Array.isArray(localDataPaths)) {
localDataPaths = [localDataPaths];
}
// Filter out files we know don't exist to avoid overhead for checking
localDataPaths = localDataPaths.filter((path) => {
return this._fsExistsCache.exists(path);
});
this.config.events.emit("eleventy.dataFiles", localDataPaths);
if (!localDataPaths.length) {
return localData;
}
let dataSource = {};
for (let path of localDataPaths) {
let dataForPath = await this.getDataValue(path, null, true);
if (!isPlainObject(dataForPath)) {
debug(
"Warning: Template and Directory data files expect an object to be returned, instead `%o` returned `%o`",
path,
dataForPath
);
} else {
// clean up data for template/directory data files only.
let cleanedDataForPath = TemplateData.cleanupData(dataForPath);
for (let key in cleanedDataForPath) {
if (dataSource.hasOwnProperty(key)) {
debugWarn(
"Local data files have conflicting data. Overwriting '%s' with data from '%s'. Previous data location was from '%s'",
key,
path,
dataSource[key]
);
}
dataSource[key] = path;
}
TemplateData.mergeDeep(this.config, localData, cleanedDataForPath);
}
}
return localData;
}
async getTemplateDirectoryData(templatePath) {
if (!this.templateDirectoryData[templatePath]) {
let localDataPaths = await this.getLocalDataPaths(templatePath);
let importedData = await this.combineLocalData(localDataPaths);
this.templateDirectoryData[templatePath] = Object.assign({}, importedData);
}
return this.templateDirectoryData[templatePath];
}
getUserDataExtensions() {
if (!this.config.dataExtensions) {
return [];
}
// returning extensions in reverse order to create proper extension order
// later added formats will override first ones
return Array.from(this.config.dataExtensions.keys()).reverse();
}
getUserDataParser(extension) {
return this.config.dataExtensions.get(extension);
}
isUserDataExtension(extension) {
return this.config.dataExtensions && this.config.dataExtensions.has(extension);
}
hasUserDataExtensions() {
return this.config.dataExtensions && this.config.dataExtensions.size > 0;
}
async _loadFileContents(path, options = {}) {
let rawInput;
let encoding = "utf8";
if ("encoding" in options) {
encoding = options.encoding;
}
try {
rawInput = await fs.promises.readFile(path, encoding);
} catch (e) {
// if file does not exist, return nothing
}
// Can return a buffer, string, etc
if (typeof rawInput === "string") {
return rawInput.trim();
}
return rawInput;
}
async _parseDataFile(path, rawImports, ignoreProcessing, parser, options = {}) {
let readFile = !("read" in options) || options.read === true;
let rawInput;
if (readFile) {
rawInput = await this._loadFileContents(path, options);
}
if (readFile && !rawInput) {
return {};
}
try {
if (readFile) {
return parser(rawInput, path);
} else {
// path as a first argument is when `read: false`
// path as a second argument is for consistency with `read: true` API
return parser(path, path);
}
} catch (e) {
throw new TemplateDataParseError(`Having trouble parsing data file ${path}`, e);
}
}
// ignoreProcessing = false for global data files
// ignoreProcessing = true for local data files
async getDataValue(path, rawImports, ignoreProcessing) {
let extension = TemplatePath.getExtension(path);
if (extension === "js" || extension === "cjs") {
// JS data file or required JSON (no preprocessing needed)
let localPath = TemplatePath.absolutePath(path);
let exists = this._fsExistsCache.exists(localPath);
// Make sure that relative lookups benefit from cache
this._fsExistsCache.markExists(path, exists);
if (!exists) {
return {};
}
let aggregateDataBench = this.benchmarks.aggregate.get("Data File");
aggregateDataBench.before();
let dataBench = this.benchmarks.data.get(`\`${path}\``);
dataBench.before();
let returnValue = EleventyRequire(localPath);
// TODO special exception for Global data `permalink.js`
// module.exports = (data) => `${data.page.filePathStem}/`; // Does not work
// module.exports = () => ((data) => `${data.page.filePathStem}/`); // Works
if (typeof returnValue === "function") {
let configApiGlobalData = await this.getInitialGlobalData();
returnValue = await returnValue(configApiGlobalData || {});
}
dataBench.after();
aggregateDataBench.after();
return returnValue;
} else if (this.isUserDataExtension(extension)) {
// Other extensions
let { parser, options } = this.getUserDataParser(extension);
return this._parseDataFile(path, rawImports, ignoreProcessing, parser, options);
} else if (extension === "json") {
// File to string, parse with JSON (preprocess)
const parser = (content) => JSON.parse(content);
return this._parseDataFile(path, rawImports, ignoreProcessing, parser);
} else {
throw new TemplateDataParseError(
`Could not find an appropriate data parser for ${path}. Do you need to add a plugin to your config file?`
);
}
}
_pushExtensionsToPaths(paths, curpath, extensions) {
for (let extension of extensions) {
paths.push(curpath + "." + extension);
}
}
_addBaseToPaths(paths, base, extensions, nonEmptySuffixesOnly = false) {
let suffixes = this.getDataFileSuffixes();
for (let suffix of suffixes) {
suffix = suffix || "";
if (nonEmptySuffixesOnly && suffix === "") {
continue;
}
// data suffix
if (suffix) {
paths.push(base + suffix + ".js");
paths.push(base + suffix + ".cjs");
}
paths.push(base + suffix + ".json"); // default: .11tydata.json
// inject user extensions
this._pushExtensionsToPaths(paths, base + suffix, extensions);
}
}
async getLocalDataPaths(templatePath) {
let paths = [];
let parsed = path.parse(templatePath);
let inputDir = TemplatePath.addLeadingDotSlash(TemplatePath.normalize(this.inputDir));
debugDev("getLocalDataPaths(%o)", templatePath);
debugDev("parsed.dir: %o", parsed.dir);
let userExtensions = this.getUserDataExtensions();
if (parsed.dir) {
let fileNameNoExt = this.extensionMap.removeTemplateExtension(parsed.base);
// default dataSuffix: .11tydata, is appended in _addBaseToPaths
debug("Using %o suffixes to find data files.", this.getDataFileSuffixes());
// Template data file paths
let filePathNoExt = parsed.dir + "/" + fileNameNoExt;
this._addBaseToPaths(paths, filePathNoExt, userExtensions);
// Directory data file paths
let allDirs = TemplatePath.getAllDirs(parsed.dir);
debugDev("allDirs: %o", allDirs);
for (let dir of allDirs) {
let lastDir = TemplatePath.getLastPathSegment(dir);
let dirPathNoExt = dir + "/" + lastDir;
if (inputDir) {
debugDev("dirStr: %o; inputDir: %o", dir, inputDir);
}
if (!inputDir || (dir.startsWith(inputDir) && dir !== inputDir)) {
if (this.config.dataFileDirBaseNameOverride) {
let indexDataFile = dir + "/" + this.config.dataFileDirBaseNameOverride;
this._addBaseToPaths(paths, indexDataFile, userExtensions, true);
} else {
this._addBaseToPaths(paths, dirPathNoExt, userExtensions);
}
}
}
// 0.11.0+ include root input dir files
// if using `docs/` as input dir, looks for docs/docs.json et al
if (inputDir) {
let lastInputDir = TemplatePath.addLeadingDotSlash(
TemplatePath.join(inputDir, TemplatePath.getLastPathSegment(inputDir))
);
// in root input dir, search for index.11tydata.json et al
if (this.config.dataFileDirBaseNameOverride) {
let indexDataFile =
TemplatePath.getDirFromFilePath(lastInputDir) +
"/" +
this.config.dataFileDirBaseNameOverride;
this._addBaseToPaths(paths, indexDataFile, userExtensions, true);
} else if (lastInputDir !== "./") {
this._addBaseToPaths(paths, lastInputDir, userExtensions);
}
}
}
debug("getLocalDataPaths(%o): %o", templatePath, paths);
return unique(paths).reverse();
}
static mergeDeep(config, target, ...source) {
if (config.dataDeepMerge) {
return TemplateData.merge(target, ...source);
} else {
return Object.assign(target, ...source);
}
}
static merge(target, ...source) {
return merge(target, ...source);
}
static cleanupData(data) {
if (isPlainObject(data) && "tags" in data) {
if (typeof data.tags === "string") {
data.tags = data.tags ? [data.tags] : [];
} else if (data.tags === null) {
data.tags = [];
}
// Deduplicate tags
data.tags = [...new Set(data.tags)];
}
return data;
}
async getServerlessPathData() {
let configApiGlobalData = await this.getInitialGlobalData();
return configApiGlobalData?.eleventy?.serverless?.path;
}
}
module.exports = TemplateData;