node.js - 并行读取多个文件（Nodejs）

Question

我有 2 个文件（或更多）1.txt 和 2.txt。
1.txt

1-a
1-b
1-c

2.txt

2-a
2-b
2-c

现在我想逐行并行读取 1.txt 和 2.txt ，并在读取后将该行写入 result.txt 。结果.txt。

1.a + 2.a
1.b + 2.b
1.c + 2.c

我怎样才能做到这一点？
编辑：这是一个简单的例子。我想处理大文件，所以我想尽可能地节省内存，我不想使用数组。

Answer 1

我认为使用异步 I/O 和尽可能少的内存（仅缓冲 readStream 在data事件中为您提供的内容）并随时输出每一行将是一个有趣的挑战。

所以，这是一种方法。这没有在非常大的文件上进行广泛测试，但在我尝试的测试用例中似乎确实有效。如果两个文件的行数不同，则在读取较短的文件时它将停止输出行。这是代码：

const LineReader = require('line-by-line');
const fs = require('fs');

class DualLineReader extends LineReader {
    constructor(filename, outputStream) {
        super(filename);
        this.myBuffer = [];
        this.output = outputStream;
    }
    setOtherReader(lr) {
        this.otherLineReader = lr;
    }

    resumeIfEmpty() {
        if (!this.myBuffer.length) {
            this.resume();
        }
    }

    // if data in both buffers, write the next line
    // call the callback when OK to call writeLine again
    writeLine(cb) {
        // if both buffers contain at least one line, output the first line in each
        if (this.myBuffer.length && this.otherLineReader.myBuffer.length) {
            let ready = this.output.write(this.myBuffer.shift() + " + " + this.otherLineReader.myBuffer.shift() + '\n');
            if (!ready) {
                // need to wait for drain event before writing any more
                this.output.once('drain', () => {
                    cb(true);
                });
            } else {
                process.nextTick(() => {
                    cb(true);
                });
            }
        } else {
            // nothing else to write at the moment
            // call the callback on next tick
            process.nextTick(() => {
                cb(false);
            });
        }
    }

    closeOutput(cb) {
        if (!this.output.closed) {
            this.output.end(cb);
        }
    }

    // loop asynchronously until no more data in buffer to write
    // call callback when done
    writeAllLines(cb = function() {}) {
        this.writeLine(more => {
            if (more) {
                this.writeAllLines();
            } else {
                // if either buffer is empty, start it flowing again
                this.resumeIfEmpty();
                this.otherLineReader.resumeIfEmpty();
                cb();
            }
        });
    }

    run(cb) {
        this.on('line', line => {
            this.myBuffer.push(line);
            this.pause();
            this.writeAllLines();
        });

        this.on('end', () => {
            this.writeAllLines(() => {
                this.close();
                this.otherLineReader.close();
                this.closeOutput(cb);
            });
        });

        this.on('error', (err) => {
            console.log(err);
            this.close();
            this.otherLineReader.close();
            this.closeOutput(() => {
                cb(err);
            });
        });

    }
}

let output = fs.createWriteStream("results.txt");
output.on('close', () => {
    this.closed = true;
});

let lr1 = new DualLineReader("file1.txt", output);
let lr2 = new DualLineReader("file2.txt", output);

lr1.setOtherReader(lr2);
lr2.setOtherReader(lr1);

function done() {
    console.log("all done");
}
lr1.run(done);
lr2.run(done);

Answer 2

花了 2 小时以上阅读 10 多篇文章以找到合适的解决方案。下面工作正常。

安装节点模块：

npm install --save n-readlines

代码：

const lineByLine = require("n-readlines");

var filePath1 = "./file1.txt", // <------ NOTE: Make sure that this file exists
    filePath2 = "./file2.txt"; // <------ NOTE: Make sure that this file exists
var file1 = new lineByLine(filePath1),
    file2 = new lineByLine(filePath2);
var line1, line2;

while (true) {
    line1 = file1.next(); // Read next line from 1st file
    if (!line1) break; // EOF

    line2 = file2.next(); // Read next line from 2nd file
    if (!line2) break; // EOF

    line1 = line1.toString("ascii").trim(); // Buffer to text, then trim spaces
    line2 = line2.toString("ascii").trim(); // Buffer to text, then trim spaces

    console.log(line1, line2);
}

Answer 3

我认为做你想做的最简单的方法是阅读所有文件并加上它们。您可以使用 ReadFileSync 函数来读取文件。我给你一个例子。希望能帮到你。

var fs = require("fs");

var file_1 = fs.readFileSync("files/1.txt");
var file_2 = fs.readFileSync("files/2.txt");

array_file_1 = file_1.toString().split("\r\n");
array_file_2 = file_2.toString().split("\r\n");

string_file = '';
array_file_1.forEach((element, index) => {
    string_file += element + array_file_2[index] + '\r\n';
});

fs.writeFileSync("files/result.txt", string_file);

console.log(string_file);