nodejs
/
node
mirror of https://github.com/nodejs/node.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fs: expose glob and globSync 

PR-URL: https://github.com/nodejs/node/pull/51912
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Chemi Atlow <chemi@atlow.co.il>
Reviewed-By: Nitzan Uziely <linkgoron@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
This commit is contained in:
Moshe Atlow 2024-03-04 01:15:35 +02:00 committed by GitHub
parent 2a33e95093
commit 151d365ad1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 443 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
doc/api/fs.md
View File

@ -1069,6 +1069,38 @@ including subdirectories and files.
When copying a directory to another directory, globs are not supported and
behavior is similar to `cp dir1/ dir2/`.
### `fsPromises.glob(pattern[, options])`
<!-- YAML
added: REPLACEME
-->
> Stability: 1 - Experimental
* `pattern` {string|string\[]}
* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* Returns: {AsyncIterator} An AsyncIterator that yields the paths of files
that match the pattern.
```mjs
import { glob } from 'node:fs/promises';
for await (const entry of glob('**/*.js'))
console.log(entry);
```
```cjs
const { glob } = require('node:fs/promises');
(async () => {
for await (const entry of glob('**/*.js'))
console.log(entry);
})();
```
### `fsPromises.lchmod(path, mode)`
<!-- YAML
@ -3073,6 +3105,44 @@ changes:
Change the file system timestamps of the object referenced by the supplied file
descriptor. See [`fs.utimes()`][].
### `fs.glob(pattern[, options], callback)`
<!-- YAML
added: REPLACEME
-->
> Stability: 1 - Experimental
* `pattern` {string|string\[]}
* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* `callback` {Function}
* `err` {Error}
* Retrieves the files matching the specified pattern.
```mjs
import { glob } from 'node:fs';
glob('**/*.js', (err, matches) => {
if (err) throw err;
console.log(matches);
});
```
```cjs
const { glob } = require('node:fs');
glob('**/*.js', (err, matches) => {
if (err) throw err;
console.log(matches);
});
```
### `fs.lchmod(path, mode, callback)`
<!-- YAML
@ -5529,6 +5599,33 @@ changes:
Synchronous version of [`fs.futimes()`][]. Returns `undefined`.
### `fs.globSync(pattern[, options])`
<!-- YAML
added: REPLACEME
-->
> Stability: 1 - Experimental
* `pattern` {string|string\[]}
* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* Returns: {string\[]} paths of files that match the pattern.
```mjs
import { globSync } from 'node:fs';
console.log(globSync('**/*.js'));
```
```cjs
const { globSync } = require('node:fs');
console.log(globSync('**/*.js'));
```
### `fs.lchmodSync(path, mode)`
<!-- YAML

36
lib/fs.js
View File

@ -86,6 +86,8 @@ const {
const { toPathIfFileURL } = require('internal/url');
const {
customPromisifyArgs: kCustomPromisifyArgsSymbol,
emitExperimentalWarning,
getLazy,
kEmptyObject,
promisify: {
custom: kCustomPromisifiedSymbol,
@ -3102,6 +3104,38 @@ function createWriteStream(path, options) {
return new WriteStream(path, options);
}
const lazyGlob = getLazy(() => require('internal/fs/glob').Glob);
function glob(pattern, options, callback) {
emitExperimentalWarning('glob');
if (typeof options === 'function') {
callback = options;
options = undefined;
}
callback = makeCallback(callback);
const Glob = lazyGlob();
// TODO: Use iterator helpers when available
(async () => {
try {
const res = [];
for await (const entry of new Glob(pattern, options).glob()) {
ArrayPrototypePush(res, entry);
}
callback(null, res);
} catch (err) {
callback(err);
}
})();
}
function globSync(pattern, options) {
emitExperimentalWarning('globSync');
const Glob = lazyGlob();
return new Glob(pattern, options).globSync();
}
module.exports = fs = {
appendFile,
appendFileSync,
@ -3135,6 +3169,8 @@ module.exports = fs = {
ftruncateSync,
futimes,
futimesSync,
glob,
globSync,
lchown,
lchownSync,
lchmod: constants.O_SYMLINK !== undefined ? lchmod : undefined,

285
lib/internal/fs/glob.js
View File

@ -1,5 +1,6 @@
'use strict';
const { lstatSync, readdirSync } = require('fs');
const { lstat, readdir } = require('fs/promises');
const { join, resolve } = require('path');
const {
@ -8,6 +9,8 @@ const {
const {
validateFunction,
validateObject,
validateString,
validateStringArray,
} = require('internal/validators');
const {
@ -18,6 +21,7 @@ const {
ArrayPrototypePop,
ArrayPrototypePush,
ArrayPrototypeSome,
PromisePrototypeThen,
SafeMap,
SafeSet,
StringPrototypeEndsWith,
@ -37,6 +41,15 @@ class Cache {
#statsCache = new SafeMap();
#readdirCache = new SafeMap();
stat(path) {
const cached = this.#statsCache.get(path);
if (cached) {
return cached;
}
const promise = PromisePrototypeThen(lstat(path), null, () => null);
this.#statsCache.set(path, promise);
return promise;
}
statSync(path) {
const cached = this.#statsCache.get(path);
if (cached) {
@ -54,6 +67,15 @@ class Cache {
addToStatCache(path, val) {
this.#statsCache.set(path, val);
}
async readdir(path) {
const cached = this.#readdirCache.get(path);
if (cached) {
return cached;
}
const promise = PromisePrototypeThen(readdir(path, { __proto__: null, withFileTypes: true }), null, () => null);
this.#readdirCache.set(path, promise);
return promise;
}
readdirSync(path) {
const cached = this.#readdirCache.get(path);
if (cached) {
@ -151,7 +173,8 @@ class Glob {
#results = new SafeSet();
#queue = [];
#subpatterns = new SafeMap();
constructor(patterns, options = kEmptyObject) {
#patterns;
constructor(pattern, options = kEmptyObject) {
validateObject(options, 'options');
const { exclude, cwd } = options;
if (exclude != null) {
@ -159,6 +182,14 @@ class Glob {
}
this.#root = cwd ?? '.';
this.#exclude = exclude;
let patterns;
if (typeof pattern === 'object') {
validateStringArray(pattern, 'patterns');
patterns = pattern;
} else {
validateString(pattern, 'patterns');
patterns = [pattern];
}
this.matchers = ArrayPrototypeMap(patterns, (pattern) => new (lazyMinimatch().Minimatch)(pattern, {
__proto__: null,
nocase: isWindows || isOSX,
@ -169,21 +200,18 @@ class Glob {
platform: process.platform,
nocaseMagicOnly: true,
}));
this.#patterns = ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set,
(pattern, i) => new Pattern(
pattern,
matcher.globParts[i],
new SafeSet().add(0),
new SafeSet(),
)));
}
globSync() {
ArrayPrototypePush(this.#queue, {
__proto__: null,
path: '.',
patterns: ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set,
(pattern, i) => new Pattern(
pattern,
matcher.globParts[i],
new SafeSet([0]),
new SafeSet(),
))),
});
ArrayPrototypePush(this.#queue, { __proto__: null, path: '.', patterns: this.#patterns });
while (this.#queue.length > 0) {
const item = ArrayPrototypePop(this.#queue);
for (let i = 0; i < item.patterns.length; i++) {
@ -216,22 +244,22 @@ class Glob {
if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
// Absolute path, go to root
this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet([1])));
this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet().add(1)));
return;
}
if (isFirst && pattern.at(0) === '') {
// Absolute path, go to root
this.#addSubpattern('/', pattern.child(new SafeSet([1])));
this.#addSubpattern('/', pattern.child(new SafeSet().add(1)));
return;
}
if (isFirst && pattern.at(0) === '..') {
// Start with .., go to parent
this.#addSubpattern('../', pattern.child(new SafeSet([1])));
this.#addSubpattern('../', pattern.child(new SafeSet().add(1)));
return;
}
if (isFirst && pattern.at(0) === '.') {
// Start with ., proceed
this.#addSubpattern('.', pattern.child(new SafeSet([1])));
this.#addSubpattern('.', pattern.child(new SafeSet().add(1)));
return;
}
@ -264,7 +292,7 @@ class Glob {
stat.name = firstPattern;
children = [stat];
} else {
children = [];
return;
}
} else {
children = this.#cache.readdirSync(fullpath);
@ -329,18 +357,18 @@ class Glob {
const parent = join(path, '..');
if (nextIndex < last) {
if (!this.#subpatterns.has(path) && !this.#cache.seen(path, pattern, nextIndex + 1)) {
this.#subpatterns.set(path, [pattern.child(new SafeSet([nextIndex + 1]))]);
this.#subpatterns.set(path, [pattern.child(new SafeSet().add(nextIndex + 1))]);
}
if (!this.#subpatterns.has(parent) && !this.#cache.seen(parent, pattern, nextIndex + 1)) {
this.#subpatterns.set(parent, [pattern.child(new SafeSet([nextIndex + 1]))]);
this.#subpatterns.set(parent, [pattern.child(new SafeSet().add(nextIndex + 1))]);
}
} else {
if (!this.#cache.seen(path, pattern, nextIndex)) {
this.#cache.add(path, pattern.child(new SafeSet([nextIndex])));
this.#cache.add(path, pattern.child(new SafeSet().add(nextIndex)));
this.#results.add(path);
}
if (!this.#cache.seen(path, pattern, nextIndex) || !this.#cache.seen(parent, pattern, nextIndex)) {
this.#cache.add(parent, pattern.child(new SafeSet([nextIndex])));
this.#cache.add(parent, pattern.child(new SafeSet().add(nextIndex)));
this.#results.add(parent);
}
}
@ -376,6 +404,219 @@ class Glob {
}
}
}
async* glob() {
ArrayPrototypePush(this.#queue, { __proto__: null, path: '.', patterns: this.#patterns });
while (this.#queue.length > 0) {
const item = ArrayPrototypePop(this.#queue);
for (let i = 0; i < item.patterns.length; i++) {
yield* this.#iterateSubpatterns(item.path, item.patterns[i]);
}
this.#subpatterns
.forEach((patterns, path) => ArrayPrototypePush(this.#queue, { __proto__: null, path, patterns }));
this.#subpatterns.clear();
}
}
async* #iterateSubpatterns(path, pattern) {
const seen = this.#cache.add(path, pattern);
if (seen) {
return;
}
const fullpath = resolve(this.#root, path);
const stat = await this.#cache.stat(fullpath);
const last = pattern.last;
const isDirectory = stat?.isDirectory() || (stat?.isSymbolicLink() && pattern.hasSeenSymlinks);
const isLast = pattern.isLast(isDirectory);
const isFirst = pattern.isFirst();
if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
// Absolute path, go to root
this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet().add(1)));
return;
}
if (isFirst && pattern.at(0) === '') {
// Absolute path, go to root
this.#addSubpattern('/', pattern.child(new SafeSet().add(1)));
return;
}
if (isFirst && pattern.at(0) === '..') {
// Start with .., go to parent
this.#addSubpattern('../', pattern.child(new SafeSet().add(1)));
return;
}
if (isFirst && pattern.at(0) === '.') {
// Start with ., proceed
this.#addSubpattern('.', pattern.child(new SafeSet().add(1)));
return;
}
if (isLast && typeof pattern.at(-1) === 'string') {
// Add result if it exists
const p = pattern.at(-1);
const stat = await this.#cache.stat(join(fullpath, p));
if (stat && (p || isDirectory)) {
const result = join(path, p);
if (!this.#results.has(result)) {
this.#results.add(result);
yield result;
}
}
if (pattern.indexes.size === 1 && pattern.indexes.has(last)) {
return;
}
} else if (isLast && pattern.at(-1) === lazyMinimatch().GLOBSTAR &&
(path !== '.' || pattern.at(0) === '.' || (last === 0 && stat))) {
// If pattern ends with **, add to results
// if path is ".", add it only if pattern starts with "." or pattern is exactly "**"
if (!this.#results.has(path)) {
this.#results.add(path);
yield path;
}
}
if (!isDirectory) {
return;
}
let children;
const firstPattern = pattern.indexes.size === 1 && pattern.at(pattern.indexes.values().next().value);
if (typeof firstPattern === 'string') {
const stat = await this.#cache.stat(join(fullpath, firstPattern));
if (stat) {
stat.name = firstPattern;
children = [stat];
} else {
return;
}
} else {
children = await this.#cache.readdir(fullpath);
}
for (let i = 0; i < children.length; i++) {
const entry = children[i];
const entryPath = join(path, entry.name);
this.#cache.addToStatCache(join(fullpath, entry.name), entry);
const subPatterns = new SafeSet();
const nSymlinks = new SafeSet();
for (const index of pattern.indexes) {
// For each child, chek potential patterns
if (this.#cache.seen(entryPath, pattern, index) || this.#cache.seen(entryPath, pattern, index + 1)) {
return;
}
const current = pattern.at(index);
const nextIndex = index + 1;
const next = pattern.at(nextIndex);
const fromSymlink = pattern.symlinks.has(index);
if (current === lazyMinimatch().GLOBSTAR) {
if (entry.name[0] === '.' || (this.#exclude && this.#exclude(entry.name))) {
continue;
}
if (!fromSymlink && entry.isDirectory()) {
// If directory, add ** to its potential patterns
subPatterns.add(index);
} else if (!fromSymlink && index === last) {
// If ** is last, add to results
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield entryPath;
}
}
// Any pattern after ** is also a potential pattern
// so we can already test it here
const nextMatches = pattern.test(nextIndex, entry.name);
if (nextMatches && nextIndex === last && !isLast) {
// If next pattern is the last one, add to results
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield entryPath;
}
} else if (nextMatches && entry.isDirectory()) {
// Pattern mached, meaning two patterns forward
// are also potential patterns
// e.g **/b/c when entry is a/b - add c to potential patterns
subPatterns.add(index + 2);
}
if ((nextMatches || pattern.at(0) === '.') &&
(entry.isDirectory() || entry.isSymbolicLink()) && !fromSymlink) {
// If pattern after ** matches, or pattern starts with "."
// and entry is a directory or symlink, add to potential patterns
subPatterns.add(nextIndex);
}
if (entry.isSymbolicLink()) {
nSymlinks.add(index);
}
if (next === '..' && entry.isDirectory()) {
// In case pattern is "**/..",
// both parent and current directory should be added to the queue
// if this is the last pattern, add to results instead
const parent = join(path, '..');
if (nextIndex < last) {
if (!this.#subpatterns.has(path) && !this.#cache.seen(path, pattern, nextIndex + 1)) {
this.#subpatterns.set(path, [pattern.child(new SafeSet().add(nextIndex + 1))]);
}
if (!this.#subpatterns.has(parent) && !this.#cache.seen(parent, pattern, nextIndex + 1)) {
this.#subpatterns.set(parent, [pattern.child(new SafeSet().add(nextIndex + 1))]);
}
} else {
if (!this.#cache.seen(path, pattern, nextIndex)) {
this.#cache.add(path, pattern.child(new SafeSet().add(nextIndex)));
if (!this.#results.has(path)) {
this.#results.add(path);
yield path;
}
}
if (!this.#cache.seen(path, pattern, nextIndex) || !this.#cache.seen(parent, pattern, nextIndex)) {
this.#cache.add(parent, pattern.child(new SafeSet().add(nextIndex)));
if (!this.#results.has(parent)) {
this.#results.add(parent);
yield parent;
}
}
}
}
}
if (typeof current === 'string') {
if (pattern.test(index, entry.name) && index !== last) {
// If current pattern matches entry name
// the next pattern is a potential pattern
subPatterns.add(nextIndex);
} else if (current === '.' && pattern.test(nextIndex, entry.name)) {
// If current pattern is ".", proceed to test next pattern
if (nextIndex === last) {
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield entryPath;
}
} else {
subPatterns.add(nextIndex + 1);
}
}
}
if (typeof current === 'object' && pattern.test(index, entry.name)) {
// If current pattern is a regex that matches entry name (e.g *.js)
// add next pattern to potential patterns, or to results if it's the last pattern
if (index === last) {
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield entryPath;
}
} else if (entry.isDirectory()) {
subPatterns.add(nextIndex);
}
}
}
if (subPatterns.size > 0) {
// If there are potential patterns, add to queue
this.#addSubpattern(entryPath, pattern.child(subPatterns, nSymlinks));
}
}
}
}
module.exports = {

11
lib/internal/fs/promises.js
View File

@ -90,10 +90,11 @@ const pathModule = require('path');
const { isAbsolute } = pathModule;
const { toPathIfFileURL } = require('internal/url');
const {
emitExperimentalWarning,
getLazy,
kEmptyObject,
lazyDOMException,
promisify,
getLazy,
} = require('internal/util');
const EventEmitter = require('events');
const { StringDecoder } = require('string_decoder');
@ -1267,11 +1268,19 @@ async function* _watch(filename, options = kEmptyObject) {
yield* watch(filename, options);
}
const lazyGlob = getLazy(() => require('internal/fs/glob').Glob);
async function* glob(pattern, options) {
emitExperimentalWarning('glob');
const Glob = lazyGlob();
yield* new Glob(pattern, options).glob();
}
module.exports = {
exports: {
access,
copyFile,
cp,
glob,
open,
opendir: promisify(opendir),
rename,

47
test/parallel/test-fs-glob.mjs
View File

@ -1,11 +1,11 @@
// Flags: --expose-internals
import * as common from '../common/index.mjs';
import tmpdir from '../common/tmpdir.js';
import { resolve, dirname, sep } from 'node:path';
import { mkdir, writeFile, symlink } from 'node:fs/promises';
import { test } from 'node:test';
import { mkdir, writeFile, symlink, glob as asyncGlob } from 'node:fs/promises';
import { glob, globSync } from 'node:fs';
import { test, describe } from 'node:test';
import { promisify } from 'node:util';
import assert from 'node:assert';
import glob from 'internal/fs/glob';
tmpdir.refresh();
@ -301,10 +301,35 @@ const patterns = {
],
};
for (const [pattern, expected] of Object.entries(patterns)) {
test(pattern, () => {
const actual = new glob.Glob([pattern], { cwd: fixtureDir }).globSync().sort();
const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
assert.deepStrictEqual(actual, normalized);
});
}
describe('glob', function() {
const promisified = promisify(glob);
for (const [pattern, expected] of Object.entries(patterns)) {
test(pattern, async () => {
const actual = (await promisified(pattern, { cwd: fixtureDir })).sort();
const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
assert.deepStrictEqual(actual, normalized);
});
}
});
describe('globSync', function() {
for (const [pattern, expected] of Object.entries(patterns)) {
test(pattern, () => {
const actual = globSync(pattern, { cwd: fixtureDir }).sort();
const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
assert.deepStrictEqual(actual, normalized);
});
}
});
describe('fsPromises glob', function() {
for (const [pattern, expected] of Object.entries(patterns)) {
test(pattern, async () => {
const actual = [];
for await (const item of asyncGlob(pattern, { cwd: fixtureDir })) actual.push(item);
actual.sort();
const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
assert.deepStrictEqual(actual, normalized);
});
}
});

1
test/parallel/test-permission-fs-supported.js
View File

@ -90,6 +90,7 @@ const ignoreList = [
...syncAndAsyncAPI('readv'),
...syncAndAsyncAPI('write'),
...syncAndAsyncAPI('writev'),
...syncAndAsyncAPI('glob'),
];
{