diff --git a/TODOS.md b/TODOS.md index 2127bd3..1cd0963 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1,7 +1,7 @@ # TODOs -1. Encrypt/decrypt the database in memory. +1. Consider giving db its own allocator 2. **db.odin:324-327** — Map iteration (`remote_set`) is non-deterministic. Same file can produce different JSON on each backup, causing spurious DB diffs. Sort remotes before storing. diff --git a/config.odin b/config.odin index 6d61ef1..22bd25d 100644 --- a/config.odin +++ b/config.odin @@ -75,7 +75,7 @@ envr_dir :: proc(config_path: string) -> string { return filepath.dir(config_path) } -data_encrypted_path :: proc(config_path: string) -> string { +data_path :: proc(config_path: string) -> string { path, _ := filepath.join([]string{envr_dir(config_path), "data.envr"}) return path } diff --git a/config_test.odin b/config_test.odin index a52bb76..e3df0fe 100644 --- a/config_test.odin +++ b/config_test.odin @@ -163,8 +163,8 @@ test_envr_dir :: proc(t: ^testing.T) { } @(test) -test_data_encrypted_path :: proc(t: ^testing.T) { - p := data_encrypted_path("/tmp/envr-fake-home-datapath/config.json") +test_data_path :: proc(t: ^testing.T) { + p := data_path("/tmp/envr-fake-home-datapath/config.json") defer delete(p) testing.expectf(t, strings.has_suffix(p, "data.envr"), "should end with data.envr, got %s", p) testing.expectf(t, strings.contains(p, ".envr"), "should contain .envr dir, got %s", p) diff --git a/db.odin b/db.odin index ad6c9f3..c3a9ffb 100644 --- a/db.odin +++ b/db.odin @@ -66,7 +66,7 @@ db_open :: proc(cfg_path: string) -> (Db, bool) { return Db{}, false } - data_path := data_encrypted_path(cfg.config_path) + data_path := data_path(cfg.config_path) _, stat_err := os.stat(data_path, context.allocator) db: ^rawptr @@ -95,32 +95,31 @@ db_open :: proc(cfg_path: string) -> (Db, bool) { } db_close :: proc(d: ^Db) { + defer sqlite.db_close(d.db) + if d.changed { - tmp_path := make_temp_path() - - if !db_vacuum_to_file(d.db, tmp_path) { - os.remove(tmp_path) - sqlite.db_close(d.db) + rc := sqlite.db_exec(d.db, "VACUUM", nil, nil, nil) + if rc != sqlite.OK { + fmt.printf("Error vacuuming database: %s\n", sqlite.db_errmsg(d.db)) return } - sqlite_data, read_err := os.read_entire_file_from_path(tmp_path, context.allocator) - os.remove(tmp_path) - if read_err != nil { - fmt.printf("Error reading vacuumed database: %v\n", read_err) - sqlite.db_close(d.db) + sz: i64 + data := sqlite.serialize(d.db, "main", &sz, 0) + if data == nil { + fmt.println("Error: failed to serialize database") return } + defer sqlite.free(data) + sqlite_data := data[:sz] encrypted, enc_ok := encrypt(sqlite_data, d.cfg.Keys[:]) - delete(sqlite_data) if !enc_ok { fmt.println("Error: encryption failed") - sqlite.db_close(d.db) return } - data_path := data_encrypted_path(d.cfg.config_path) + data_path := data_path(d.cfg.config_path) envr_d := envr_dir(d.cfg.config_path) os.mkdir_all(envr_d) @@ -128,13 +127,11 @@ db_close :: proc(d: ^Db) { delete(encrypted) if write_err != nil { fmt.printf("Error writing encrypted database: %v\n", write_err) - sqlite.db_close(d.db) return } d.changed = false } - sqlite.db_close(d.db) } // Caller is responsible for calling: @@ -192,22 +189,12 @@ db_list :: proc(d: ^Db, allocator := context.allocator) -> (results: [dynamic]En return } -db_vacuum_to_file :: proc(db: ^rawptr, path: string) -> bool { - b: strings.Builder - strings.builder_init(&b) - defer strings.builder_destroy(&b) - fmt.sbprintf(&b, "VACUUM INTO '%s'", path) - rc := sqlite.db_exec(db, to_cstring(&b), nil, nil, nil) - if rc != sqlite.OK { - fmt.printf("Error vacuuming database: %s\n", sqlite.db_errmsg(db)) - return false - } - return true -} - db_restore_from_encrypted :: proc(db: ^rawptr, cfg: Config) -> bool { - data_path := data_encrypted_path(cfg.config_path) - encrypted_data, read_err := os.read_entire_file_from_path(data_path, context.temp_allocator) + encrypted_data, read_err := os.read_entire_file_from_path( + data_path(cfg.config_path), + context.allocator, + ) + defer delete(encrypted_data) if read_err != nil { fmt.printf("Error reading encrypted database: %v\n", read_err) return false @@ -220,50 +207,32 @@ db_restore_from_encrypted :: proc(db: ^rawptr, cfg: Config) -> bool { } defer delete(plaintext) - tmp_path := make_temp_path() - write_err := os.write_entire_file(tmp_path, plaintext) - if write_err != nil { - fmt.printf("Error writing temp database: %v\n", write_err) + n := i64(len(plaintext)) + buf := sqlite.malloc64(n) + if buf == nil { + fmt.println("Error: failed to allocate buffer for deserialization") return false } - defer os.remove(tmp_path) + copy(buf[:len(plaintext)], plaintext) - if !db_attach_and_copy(db, tmp_path) { - return false - } - - return true -} - -db_attach_and_copy :: proc(mem_db: ^rawptr, src_path: string) -> bool { - b: strings.Builder - strings.builder_init(&b) - defer strings.builder_destroy(&b) - fmt.sbprintf(&b, "ATTACH DATABASE '%s' AS source", src_path) - - rc := sqlite.db_exec(mem_db, to_cstring(&b), nil, nil, nil) - if rc != sqlite.OK { - fmt.printf("Error attaching database: %s\n", sqlite.db_errmsg(mem_db)) - return false - } - - rc = sqlite.db_exec( - mem_db, - "INSERT INTO main.envr_env_files SELECT * FROM source.envr_env_files", - nil, - nil, - nil, + rc := sqlite.deserialize( + db, + "main", + buf, + n, + n, + sqlite.DESERIALIZE_FREEONCLOSE | sqlite.DESERIALIZE_RESIZEABLE, ) if rc != sqlite.OK { - fmt.printf("Error copying data: %s\n", sqlite.db_errmsg(mem_db)) - sqlite.db_exec(mem_db, "DETACH DATABASE source", nil, nil, nil) + sqlite.free(buf) + fmt.printf("Error deserializing database: %s\n", sqlite.db_errmsg(db)) return false } - sqlite.db_exec(mem_db, "DETACH DATABASE source", nil, nil, nil) return true } + get_git_remotes :: proc(dir: string) -> [dynamic]string { remotes: [dynamic]string remote_set: map[string]bool diff --git a/db_integration_test.odin b/db_integration_test.odin index 691a70c..2d04f14 100644 --- a/db_integration_test.odin +++ b/db_integration_test.odin @@ -136,7 +136,7 @@ test_encrypt_write_read_decrypt :: proc(t: ^testing.T) { } @(test) -test_decrypt_then_attach_sqlite :: proc(t: ^testing.T) { +test_decrypt_then_deserialize_sqlite :: proc(t: ^testing.T) { cfg := fixture_config() defer { delete(cfg.Keys) @@ -164,14 +164,6 @@ test_decrypt_then_attach_sqlite :: proc(t: ^testing.T) { } defer delete(plaintext) - tmp_db_path := fmt.tprintf("/tmp/envr-test-attach-%d.db", os.get_pid()) - write_err := os.write_entire_file(tmp_db_path, plaintext) - testing.expectf(t, write_err == nil, "failed to write temp db: %v", write_err) - if write_err != nil { - return - } - defer os.remove(tmp_db_path) - mem_db: ^rawptr rc := sqlite.db_open(":memory:", &mem_db) testing.expectf(t, rc == sqlite.OK, "failed to open in-memory db") @@ -180,12 +172,25 @@ test_decrypt_then_attach_sqlite :: proc(t: ^testing.T) { } defer sqlite.db_close(mem_db) - create_sql: cstring = "CREATE TABLE IF NOT EXISTS envr_env_files (path TEXT PRIMARY KEY NOT NULL, remotes TEXT, sha256 TEXT NOT NULL, contents TEXT NOT NULL)" - rc = sqlite.db_exec(mem_db, create_sql, nil, nil, nil) - testing.expect(t, rc == sqlite.OK, "failed to create table") + n := i64(len(plaintext)) + buf := sqlite.malloc64(n) + testing.expect(t, buf != nil, "malloc64 should succeed") + if buf == nil do return + copy(buf[:len(plaintext)], plaintext) - attach_ok := db_attach_and_copy(mem_db, tmp_db_path) - testing.expect(t, attach_ok, "failed to attach and copy") + rc = sqlite.deserialize( + mem_db, + "main", + buf, + n, + n, + sqlite.DESERIALIZE_FREEONCLOSE | sqlite.DESERIALIZE_RESIZEABLE, + ) + testing.expect(t, rc == sqlite.OK, "deserialize should succeed") + if rc != sqlite.OK { + sqlite.free(buf) + return + } sql: cstring = "SELECT path FROM envr_env_files" stmt: ^rawptr diff --git a/db_test.odin b/db_test.odin index 6077e0a..ba0587a 100644 --- a/db_test.odin +++ b/db_test.odin @@ -215,7 +215,7 @@ test_db_delete_sets_changed :: proc(t: ^testing.T) { } @(test) -test_db_vacuum_to_file :: proc(t: ^testing.T) { +test_db_serialize :: proc(t: ^testing.T) { d, ok := make_test_db() testing.expect(t, ok, "failed to create test db") if !ok do return @@ -225,20 +225,13 @@ test_db_vacuum_to_file :: proc(t: ^testing.T) { defer delete(f.Remotes) db_insert(&d, f) - vacuum_path := fmt.tprintf("/tmp/envr-test-vacuum-%d.db", os.get_pid()) - defer os.remove(vacuum_path) + sz: i64 + data := sqlite.serialize(d.db, "main", &sz, 0) + testing.expect(t, data != nil, "serialize should return non-nil") + if data == nil do return + defer sqlite.free(data) - testing.expect(t, db_vacuum_to_file(d.db, vacuum_path), "vacuum should succeed") - - info, stat_err := os.stat(vacuum_path, context.allocator) - defer os.file_info_delete(info, context.allocator) - testing.expect(t, stat_err == nil, "vacuumed file should exist") - - data, read_err := os.read_entire_file_from_path(vacuum_path, context.allocator) - testing.expect(t, read_err == nil, "should read vacuumed file") - defer delete(data) - - testing.expect(t, len(data) > 0, "vacuumed file should be non-empty") + testing.expect(t, sz > 0, "serialized size should be > 0") } @(test) diff --git a/flake.nix b/flake.nix index 983cfc8..20dfdf3 100644 --- a/flake.nix +++ b/flake.nix @@ -11,11 +11,12 @@ }; outputs = - inputs@{ flake-parts - , nixpkgs - , nixpkgs-unstable - , self - , treefmt-nix + inputs@{ + flake-parts, + nixpkgs, + nixpkgs-unstable, + self, + treefmt-nix, }: flake-parts.lib.mkFlake { inherit inputs; } { imports = [ @@ -29,7 +30,18 @@ ]; perSystem = - { pkgs, system, inputs', ... }: { + { + pkgs, + system, + inputs', + ... + }: + let + mysqlite = pkgs.sqlite.overrideAttrs (old: { + configureFlags = (old.configureFlags or [ ]) ++ [ "--enable-deserialize" ]; + }); + in + { _module.args.pkgs = import nixpkgs { inherit system; config.allowUnfree = true; @@ -64,7 +76,7 @@ buildInputs = [ pkgs.libsodium - pkgs.sqlite + mysqlite ]; buildPhase = '' @@ -87,7 +99,7 @@ nushell libsodium - sqlite + mysqlite unstable.odin unstable.ols diff --git a/sqlite/sqlite.odin b/sqlite/sqlite.odin index 9d1e463..45c7437 100644 --- a/sqlite/sqlite.odin +++ b/sqlite/sqlite.odin @@ -8,6 +8,9 @@ OK :: 0 ROW :: 100 DONE :: 101 +DESERIALIZE_FREEONCLOSE :: 1 +DESERIALIZE_RESIZEABLE :: 2 + foreign lib { @(link_name="sqlite3_open") db_open :: proc(filename: cstring, ppDb: ^^rawptr) -> c.int --- @@ -31,4 +34,12 @@ foreign lib { bind_text :: proc(stmt: ^rawptr, idx: c.int, val: cstring, n: c.int, destructor: rawptr) -> c.int --- @(link_name="sqlite3_changes") changes :: proc(db: ^rawptr) -> c.int --- + @(link_name="sqlite3_serialize") + serialize :: proc(db: ^rawptr, zSchema: cstring, piSize: ^i64, mFlags: u32) -> [^]u8 --- + @(link_name="sqlite3_deserialize") + deserialize :: proc(db: ^rawptr, zSchema: cstring, pData: [^]u8, szDb: i64, szBuf: i64, mFlags: u32) -> c.int --- + @(link_name="sqlite3_malloc64") + malloc64 :: proc(n: i64) -> [^]u8 --- + @(link_name="sqlite3_free") + free :: proc(p: rawptr) --- }