sqlite, test: expose sqlite online backup api

PR-URL: https://github.com/nodejs/node/pull/56253
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Vinícius Lourenço Claro Cardoso <contact@viniciusl.com.br>
This commit is contained in:
Edy Silva
2025-02-05 19:26:28 -03:00
committed by GitHub
parent 436723282d
commit 16dc29d4b6
6 changed files with 626 additions and 0 deletions

View File

@@ -526,6 +526,63 @@ exception.
| `TEXT` | {string} |
| `BLOB` | {TypedArray} or {DataView} |
## `sqlite.backup(sourceDb, destination[, options])`
<!-- YAML
added: REPLACEME
-->
* `sourceDb` {DatabaseSync} The database to backup. The source database must be open.
* `destination` {string} The path where the backup will be created. If the file already exists, the contents will be
overwritten.
* `options` {Object} Optional configuration for the backup. The
following properties are supported:
* `source` {string} Name of the source database. This can be `'main'` (the default primary database) or any other
database that have been added with [`ATTACH DATABASE`][] **Default:** `'main'`.
* `target` {string} Name of the target database. This can be `'main'` (the default primary database) or any other
database that have been added with [`ATTACH DATABASE`][] **Default:** `'main'`.
* `rate` {number} Number of pages to be transmitted in each batch of the backup. **Default:** `100`.
* `progress` {Function} Callback function that will be called with the number of pages copied and the total number of
pages.
* Returns: {Promise} A promise that resolves when the backup is completed and rejects if an error occurs.
This method makes a database backup. This method abstracts the [`sqlite3_backup_init()`][], [`sqlite3_backup_step()`][]
and [`sqlite3_backup_finish()`][] functions.
The backed-up database can be used normally during the backup process. Mutations coming from the same connection - same
{DatabaseSync} - object will be reflected in the backup right away. However, mutations from other connections will cause
the backup process to restart.
```cjs
const { backup, DatabaseSync } = require('node:sqlite');
(async () => {
const sourceDb = new DatabaseSync('source.db');
const totalPagesTransferred = await backup(sourceDb, 'backup.db', {
rate: 1, // Copy one page at a time.
progress: ({ totalPages, remainingPages }) => {
console.log('Backup in progress', { totalPages, remainingPages });
},
});
console.log('Backup completed', totalPagesTransferred);
})();
```
```mjs
import { backup, DatabaseSync } from 'node:sqlite';
const sourceDb = new DatabaseSync('source.db');
const totalPagesTransferred = await backup(sourceDb, 'backup.db', {
rate: 1, // Copy one page at a time.
progress: ({ totalPages, remainingPages }) => {
console.log('Backup in progress', { totalPages, remainingPages });
},
});
console.log('Backup completed', totalPagesTransferred);
```
## `sqlite.constants`
<!-- YAML
@@ -609,6 +666,9 @@ resolution handler passed to [`database.applyChangeset()`][]. See also
[`SQLITE_DIRECTONLY`]: https://www.sqlite.org/c3ref/c_deterministic.html
[`SQLITE_MAX_FUNCTION_ARG`]: https://www.sqlite.org/limits.html#max_function_arg
[`database.applyChangeset()`]: #databaseapplychangesetchangeset-options
[`sqlite3_backup_finish()`]: https://www.sqlite.org/c3ref/backup_finish.html#sqlite3backupfinish
[`sqlite3_backup_init()`]: https://www.sqlite.org/c3ref/backup_finish.html#sqlite3backupinit
[`sqlite3_backup_step()`]: https://www.sqlite.org/c3ref/backup_finish.html#sqlite3backupstep
[`sqlite3_changes64()`]: https://www.sqlite.org/c3ref/changes.html
[`sqlite3_close_v2()`]: https://www.sqlite.org/c3ref/close.html
[`sqlite3_create_function_v2()`]: https://www.sqlite.org/c3ref/create_function.html

View File

@@ -77,6 +77,7 @@
V(asn1curve_string, "asn1Curve") \
V(async_ids_stack_string, "async_ids_stack") \
V(attributes_string, "attributes") \
V(backup_string, "backup") \
V(base_string, "base") \
V(base_url_string, "baseURL") \
V(bits_string, "bits") \
@@ -302,6 +303,7 @@
V(primordials_string, "primordials") \
V(priority_string, "priority") \
V(process_string, "process") \
V(progress_string, "progress") \
V(promise_string, "promise") \
V(protocol_string, "protocol") \
V(prototype_string, "prototype") \
@@ -316,6 +318,7 @@
V(reason_string, "reason") \
V(refresh_string, "refresh") \
V(regexp_string, "regexp") \
V(remaining_pages_string, "remainingPages") \
V(rename_string, "rename") \
V(replacement_string, "replacement") \
V(required_module_facade_url_string, \
@@ -369,6 +372,7 @@
V(time_to_first_byte_sent_string, "timeToFirstByteSent") \
V(time_to_first_header_string, "timeToFirstHeader") \
V(tls_ticket_string, "tlsTicket") \
V(total_pages_string, "totalPages") \
V(transfer_string, "transfer") \
V(transfer_unsupported_type_str, \
"Cannot transfer object of unsupported type.") \

View File

@@ -8,6 +8,7 @@
#include "node_errors.h"
#include "node_mem-inl.h"
#include "sqlite3.h"
#include "threadpoolwork-inl.h"
#include "util-inl.h"
#include <cinttypes>
@@ -29,6 +30,7 @@ using v8::FunctionCallback;
using v8::FunctionCallbackInfo;
using v8::FunctionTemplate;
using v8::Global;
using v8::HandleScope;
using v8::Int32;
using v8::Integer;
using v8::Isolate;
@@ -40,6 +42,7 @@ using v8::NewStringType;
using v8::Null;
using v8::Number;
using v8::Object;
using v8::Promise;
using v8::SideEffectType;
using v8::String;
using v8::TryCatch;
@@ -81,6 +84,23 @@ inline MaybeLocal<Object> CreateSQLiteError(Isolate* isolate,
return e;
}
inline MaybeLocal<Object> CreateSQLiteError(Isolate* isolate, int errcode) {
const char* errstr = sqlite3_errstr(errcode);
Local<String> js_errmsg;
Local<Object> e;
Environment* env = Environment::GetCurrent(isolate);
if (!String::NewFromUtf8(isolate, errstr).ToLocal(&js_errmsg) ||
!CreateSQLiteError(isolate, errstr).ToLocal(&e) ||
e->Set(env->context(),
env->errcode_string(),
Integer::New(isolate, errcode))
.IsNothing() ||
e->Set(env->context(), env->errstr_string(), js_errmsg).IsNothing()) {
return MaybeLocal<Object>();
}
return e;
}
inline MaybeLocal<Object> CreateSQLiteError(Isolate* isolate, sqlite3* db) {
int errcode = sqlite3_extended_errcode(db);
const char* errstr = sqlite3_errstr(errcode);
@@ -137,6 +157,169 @@ inline void THROW_ERR_SQLITE_ERROR(Isolate* isolate, int errcode) {
}
}
class BackupJob : public ThreadPoolWork {
public:
explicit BackupJob(Environment* env,
DatabaseSync* source,
Local<Promise::Resolver> resolver,
std::string source_db,
std::string destination_name,
std::string dest_db,
int pages,
Local<Function> progressFunc)
: ThreadPoolWork(env, "node_sqlite3.BackupJob"),
env_(env),
source_(source),
pages_(pages),
source_db_(source_db),
destination_name_(destination_name),
dest_db_(dest_db) {
resolver_.Reset(env->isolate(), resolver);
progressFunc_.Reset(env->isolate(), progressFunc);
}
void ScheduleBackup() {
Isolate* isolate = env()->isolate();
HandleScope handle_scope(isolate);
backup_status_ = sqlite3_open_v2(destination_name_.c_str(),
&dest_,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
nullptr);
Local<Promise::Resolver> resolver =
Local<Promise::Resolver>::New(env()->isolate(), resolver_);
if (backup_status_ != SQLITE_OK) {
HandleBackupError(resolver);
return;
}
backup_ = sqlite3_backup_init(
dest_, dest_db_.c_str(), source_->Connection(), source_db_.c_str());
if (backup_ == nullptr) {
HandleBackupError(resolver);
return;
}
this->ScheduleWork();
}
void DoThreadPoolWork() override {
backup_status_ = sqlite3_backup_step(backup_, pages_);
}
void AfterThreadPoolWork(int status) override {
HandleScope handle_scope(env()->isolate());
Local<Promise::Resolver> resolver =
Local<Promise::Resolver>::New(env()->isolate(), resolver_);
if (!(backup_status_ == SQLITE_OK || backup_status_ == SQLITE_DONE ||
backup_status_ == SQLITE_BUSY || backup_status_ == SQLITE_LOCKED)) {
HandleBackupError(resolver, backup_status_);
return;
}
int total_pages = sqlite3_backup_pagecount(backup_);
int remaining_pages = sqlite3_backup_remaining(backup_);
if (remaining_pages != 0) {
Local<Function> fn =
Local<Function>::New(env()->isolate(), progressFunc_);
if (!fn.IsEmpty()) {
Local<Object> progress_info = Object::New(env()->isolate());
if (progress_info
->Set(env()->context(),
env()->total_pages_string(),
Integer::New(env()->isolate(), total_pages))
.IsNothing() ||
progress_info
->Set(env()->context(),
env()->remaining_pages_string(),
Integer::New(env()->isolate(), remaining_pages))
.IsNothing()) {
return;
}
Local<Value> argv[] = {progress_info};
TryCatch try_catch(env()->isolate());
fn->Call(env()->context(), Null(env()->isolate()), 1, argv)
.FromMaybe(Local<Value>());
if (try_catch.HasCaught()) {
Finalize();
resolver->Reject(env()->context(), try_catch.Exception()).ToChecked();
return;
}
}
// There's still work to do
this->ScheduleWork();
return;
}
if (backup_status_ != SQLITE_DONE) {
HandleBackupError(resolver);
return;
}
Finalize();
resolver
->Resolve(env()->context(), Integer::New(env()->isolate(), total_pages))
.ToChecked();
}
void Finalize() {
Cleanup();
source_->RemoveBackup(this);
}
void Cleanup() {
if (backup_) {
sqlite3_backup_finish(backup_);
backup_ = nullptr;
}
if (dest_) {
backup_status_ = sqlite3_errcode(dest_);
sqlite3_close_v2(dest_);
dest_ = nullptr;
}
}
private:
void HandleBackupError(Local<Promise::Resolver> resolver) {
Local<Object> e;
if (!CreateSQLiteError(env()->isolate(), dest_).ToLocal(&e)) {
Finalize();
return;
}
Finalize();
resolver->Reject(env()->context(), e).ToChecked();
}
void HandleBackupError(Local<Promise::Resolver> resolver, int errcode) {
Local<Object> e;
if (!CreateSQLiteError(env()->isolate(), errcode).ToLocal(&e)) {
Finalize();
return;
}
Finalize();
resolver->Reject(env()->context(), e).ToChecked();
}
Environment* env() const { return env_; }
Environment* env_;
DatabaseSync* source_;
Global<Promise::Resolver> resolver_;
Global<Function> progressFunc_;
sqlite3* dest_ = nullptr;
sqlite3_backup* backup_ = nullptr;
int pages_;
int backup_status_;
std::string source_db_;
std::string destination_name_;
std::string dest_db_;
};
UserDefinedFunction::UserDefinedFunction(Environment* env,
Local<Function> fn,
DatabaseSync* db,
@@ -279,6 +462,14 @@ DatabaseSync::DatabaseSync(Environment* env,
}
}
void DatabaseSync::AddBackup(BackupJob* job) {
backups_.insert(job);
}
void DatabaseSync::RemoveBackup(BackupJob* job) {
backups_.erase(job);
}
void DatabaseSync::DeleteSessions() {
// all attached sessions need to be deleted before the database is closed
// https://www.sqlite.org/session/sqlite3session_create.html
@@ -289,6 +480,8 @@ void DatabaseSync::DeleteSessions() {
}
DatabaseSync::~DatabaseSync() {
FinalizeBackups();
if (IsOpen()) {
FinalizeStatements();
DeleteSessions();
@@ -353,6 +546,14 @@ bool DatabaseSync::Open() {
return true;
}
void DatabaseSync::FinalizeBackups() {
for (auto backup : backups_) {
backup->Cleanup();
}
backups_.clear();
}
void DatabaseSync::FinalizeStatements() {
for (auto stmt : statements_) {
stmt->Finalize();
@@ -772,6 +973,117 @@ void DatabaseSync::CreateSession(const FunctionCallbackInfo<Value>& args) {
args.GetReturnValue().Set(session->object());
}
void Backup(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
if (args.Length() < 1 || !args[0]->IsObject()) {
THROW_ERR_INVALID_ARG_TYPE(env->isolate(),
"The \"sourceDb\" argument must be an object.");
return;
}
DatabaseSync* db;
ASSIGN_OR_RETURN_UNWRAP(&db, args[0].As<Object>());
THROW_AND_RETURN_ON_BAD_STATE(env, !db->IsOpen(), "database is not open");
if (!args[1]->IsString()) {
THROW_ERR_INVALID_ARG_TYPE(
env->isolate(), "The \"destination\" argument must be a string.");
return;
}
int rate = 100;
std::string source_db = "main";
std::string dest_db = "main";
Utf8Value dest_path(env->isolate(), args[1].As<String>());
Local<Function> progressFunc = Local<Function>();
if (args.Length() > 2) {
if (!args[2]->IsObject()) {
THROW_ERR_INVALID_ARG_TYPE(env->isolate(),
"The \"options\" argument must be an object.");
return;
}
Local<Object> options = args[2].As<Object>();
Local<Value> rate_v;
if (!options->Get(env->context(), env->rate_string()).ToLocal(&rate_v)) {
return;
}
if (!rate_v->IsUndefined()) {
if (!rate_v->IsInt32()) {
THROW_ERR_INVALID_ARG_TYPE(
env->isolate(),
"The \"options.rate\" argument must be an integer.");
return;
}
rate = rate_v.As<Int32>()->Value();
}
Local<Value> source_v;
if (!options->Get(env->context(), env->source_string())
.ToLocal(&source_v)) {
return;
}
if (!source_v->IsUndefined()) {
if (!source_v->IsString()) {
THROW_ERR_INVALID_ARG_TYPE(
env->isolate(),
"The \"options.source\" argument must be a string.");
return;
}
source_db = Utf8Value(env->isolate(), source_v.As<String>()).ToString();
}
Local<Value> target_v;
if (!options->Get(env->context(), env->target_string())
.ToLocal(&target_v)) {
return;
}
if (!target_v->IsUndefined()) {
if (!target_v->IsString()) {
THROW_ERR_INVALID_ARG_TYPE(
env->isolate(),
"The \"options.target\" argument must be a string.");
return;
}
dest_db = Utf8Value(env->isolate(), target_v.As<String>()).ToString();
}
Local<Value> progress_v;
if (!options->Get(env->context(), env->progress_string())
.ToLocal(&progress_v)) {
return;
}
if (!progress_v->IsUndefined()) {
if (!progress_v->IsFunction()) {
THROW_ERR_INVALID_ARG_TYPE(
env->isolate(),
"The \"options.progress\" argument must be a function.");
return;
}
progressFunc = progress_v.As<Function>();
}
}
Local<Promise::Resolver> resolver;
if (!Promise::Resolver::New(env->context()).ToLocal(&resolver)) {
return;
}
args.GetReturnValue().Set(resolver->GetPromise());
BackupJob* job = new BackupJob(
env, db, resolver, source_db, *dest_path, dest_db, rate, progressFunc);
db->AddBackup(job);
job->ScheduleBackup();
}
// the reason for using static functions here is that SQLite needs a
// function pointer
static std::function<int(int)> conflictCallback;
@@ -1803,6 +2115,14 @@ static void Initialize(Local<Object> target,
StatementSync::GetConstructorTemplate(env));
target->Set(context, env->constants_string(), constants).Check();
Local<Function> backup_function;
if (!Function::New(context, Backup).ToLocal(&backup_function)) {
return;
}
target->Set(context, env->backup_string(), backup_function).Check();
}
} // namespace sqlite

View File

@@ -43,6 +43,7 @@ class DatabaseOpenConfiguration {
};
class StatementSync;
class BackupJob;
class DatabaseSync : public BaseObject {
public:
@@ -64,6 +65,9 @@ class DatabaseSync : public BaseObject {
const v8::FunctionCallbackInfo<v8::Value>& args);
static void LoadExtension(const v8::FunctionCallbackInfo<v8::Value>& args);
void FinalizeStatements();
void RemoveBackup(BackupJob* backup);
void AddBackup(BackupJob* backup);
void FinalizeBackups();
void UntrackStatement(StatementSync* statement);
bool IsOpen();
sqlite3* Connection();
@@ -89,6 +93,7 @@ class DatabaseSync : public BaseObject {
sqlite3* connection_;
bool ignore_next_sqlite_error_;
std::set<BackupJob*> backups_;
std::set<sqlite3_session*> sessions_;
std::unordered_set<StatementSync*> statements_;

View File

@@ -0,0 +1,236 @@
import '../common/index.mjs';
import tmpdir from '../common/tmpdir.js';
import { join } from 'node:path';
import { backup, DatabaseSync } from 'node:sqlite';
import { describe, test } from 'node:test';
import { writeFileSync } from 'node:fs';
let cnt = 0;
tmpdir.refresh();
function nextDb() {
return join(tmpdir.path, `database-${cnt++}.db`);
}
function makeSourceDb() {
const database = new DatabaseSync(':memory:');
database.exec(`
CREATE TABLE data(
key INTEGER PRIMARY KEY,
value TEXT
) STRICT
`);
const insert = database.prepare('INSERT INTO data (key, value) VALUES (?, ?)');
for (let i = 1; i <= 2; i++) {
insert.run(i, `value-${i}`);
}
return database;
}
describe('backup()', () => {
test('throws if the source database is not provided', (t) => {
t.assert.throws(() => {
backup();
}, {
code: 'ERR_INVALID_ARG_TYPE',
message: 'The "sourceDb" argument must be an object.'
});
});
test('throws if path is not a string', (t) => {
const database = makeSourceDb();
t.assert.throws(() => {
backup(database);
}, {
code: 'ERR_INVALID_ARG_TYPE',
message: 'The "destination" argument must be a string.'
});
t.assert.throws(() => {
backup(database, {});
}, {
code: 'ERR_INVALID_ARG_TYPE',
message: 'The "destination" argument must be a string.'
});
});
test('throws if options is not an object', (t) => {
const database = makeSourceDb();
t.assert.throws(() => {
backup(database, 'hello.db', 'invalid');
}, {
code: 'ERR_INVALID_ARG_TYPE',
message: 'The "options" argument must be an object.'
});
});
test('throws if any of provided options is invalid', (t) => {
const database = makeSourceDb();
t.assert.throws(() => {
backup(database, 'hello.db', {
source: 42
});
}, {
code: 'ERR_INVALID_ARG_TYPE',
message: 'The "options.source" argument must be a string.'
});
t.assert.throws(() => {
backup(database, 'hello.db', {
target: 42
});
}, {
code: 'ERR_INVALID_ARG_TYPE',
message: 'The "options.target" argument must be a string.'
});
t.assert.throws(() => {
backup(database, 'hello.db', {
rate: 'invalid'
});
}, {
code: 'ERR_INVALID_ARG_TYPE',
message: 'The "options.rate" argument must be an integer.'
});
t.assert.throws(() => {
backup(database, 'hello.db', {
progress: 'invalid'
});
}, {
code: 'ERR_INVALID_ARG_TYPE',
message: 'The "options.progress" argument must be a function.'
});
});
});
test('database backup', async (t) => {
const progressFn = t.mock.fn();
const database = makeSourceDb();
const destDb = nextDb();
await backup(database, destDb, {
rate: 1,
progress: progressFn,
});
const backupDb = new DatabaseSync(destDb);
const rows = backupDb.prepare('SELECT * FROM data').all();
// The source database has two pages - using the default page size -,
// so the progress function should be called once (the last call is not made since
// the promise resolves)
t.assert.strictEqual(progressFn.mock.calls.length, 1);
t.assert.deepStrictEqual(progressFn.mock.calls[0].arguments, [{ totalPages: 2, remainingPages: 1 }]);
t.assert.deepStrictEqual(rows, [
{ __proto__: null, key: 1, value: 'value-1' },
{ __proto__: null, key: 2, value: 'value-2' },
]);
t.after(() => {
database.close();
backupDb.close();
});
});
test('database backup in a single call', async (t) => {
const progressFn = t.mock.fn();
const database = makeSourceDb();
const destDb = nextDb();
// Let rate to be default (100) to backup in a single call
await backup(database, destDb, {
progress: progressFn,
});
const backupDb = new DatabaseSync(destDb);
const rows = backupDb.prepare('SELECT * FROM data').all();
t.assert.strictEqual(progressFn.mock.calls.length, 0);
t.assert.deepStrictEqual(rows, [
{ __proto__: null, key: 1, value: 'value-1' },
{ __proto__: null, key: 2, value: 'value-2' },
]);
t.after(() => {
database.close();
backupDb.close();
});
});
test('throws exception when trying to start backup from a closed database', (t) => {
t.assert.throws(() => {
const database = new DatabaseSync(':memory:');
database.close();
backup(database, 'backup.db');
}, {
code: 'ERR_INVALID_STATE',
message: 'database is not open'
});
});
test('database backup fails when dest file is not writable', async (t) => {
const readonlyDestDb = nextDb();
writeFileSync(readonlyDestDb, '', { mode: 0o444 });
const database = makeSourceDb();
await t.assert.rejects(async () => {
await backup(database, readonlyDestDb);
}, {
code: 'ERR_SQLITE_ERROR',
message: 'attempt to write a readonly database'
});
});
test('backup fails when progress function throws', async (t) => {
const database = makeSourceDb();
const destDb = nextDb();
const progressFn = t.mock.fn(() => {
throw new Error('progress error');
});
await t.assert.rejects(async () => {
await backup(database, destDb, {
rate: 1,
progress: progressFn,
});
}, {
message: 'progress error'
});
});
test('backup fails when source db is invalid', async (t) => {
const database = makeSourceDb();
const destDb = nextDb();
await t.assert.rejects(async () => {
await backup(database, destDb, {
rate: 1,
source: 'invalid',
});
}, {
message: 'unknown database invalid'
});
});
test('backup fails when destination cannot be opened', async (t) => {
const database = makeSourceDb();
await t.assert.rejects(async () => {
await backup(database, `${tmpdir.path}/invalid/backup.db`);
}, {
message: 'unable to open database file'
});
});

View File

@@ -112,6 +112,7 @@ const customTypesMap = {
'Channel': 'diagnostics_channel.html#class-channel',
'TracingChannel': 'diagnostics_channel.html#class-tracingchannel',
'DatabaseSync': 'sqlite.html#class-databasesync',
'Domain': 'domain.html#class-domain',
'errors.Error': 'errors.html#class-error',