feat: All encryption/decryption now happens in-memory.

Release-as: v0.3.0

TODOS.md

@@ -1,7 +1,7 @@
 # TODOs
 
 
-1. Encrypt/decrypt the database in memory.
+1. Consider giving db its own allocator
 
 2. **db.odin:324-327** — Map iteration (`remote_set`) is non-deterministic. Same file can produce different JSON on each backup, causing spurious DB diffs. Sort remotes before storing.
 

config.odin

@@ -75,7 +75,7 @@ envr_dir :: proc(config_path: string) -> string {
 	return filepath.dir(config_path)
 }
 
-data_encrypted_path :: proc(config_path: string) -> string {
+data_path :: proc(config_path: string) -> string {
 	path, _ := filepath.join([]string{envr_dir(config_path), "data.envr"})
 	return path
 }

config_test.odin

@@ -163,8 +163,8 @@ test_envr_dir :: proc(t: ^testing.T) {
 }
 
 @(test)
-test_data_encrypted_path :: proc(t: ^testing.T) {
-	p := data_encrypted_path("/tmp/envr-fake-home-datapath/config.json")
+test_data_path :: proc(t: ^testing.T) {
+	p := data_path("/tmp/envr-fake-home-datapath/config.json")
 	defer delete(p)
 	testing.expectf(t, strings.has_suffix(p, "data.envr"), "should end with data.envr, got %s", p)
 	testing.expectf(t, strings.contains(p, ".envr"), "should contain .envr dir, got %s", p)

db.odin

@@ -66,7 +66,7 @@ db_open :: proc(cfg_path: string) -> (Db, bool) {
 		return Db{}, false
 	}
 
-	data_path := data_encrypted_path(cfg.config_path)
+	data_path := data_path(cfg.config_path)
 	_, stat_err := os.stat(data_path, context.allocator)
 
 	db: ^rawptr
@@ -95,32 +95,31 @@ db_open :: proc(cfg_path: string) -> (Db, bool) {
 }
 
 db_close :: proc(d: ^Db) {
+	defer sqlite.db_close(d.db)
+
 	if d.changed {
-		tmp_path := make_temp_path()
-
-		if !db_vacuum_to_file(d.db, tmp_path) {
-			os.remove(tmp_path)
-			sqlite.db_close(d.db)
+		rc := sqlite.db_exec(d.db, "VACUUM", nil, nil, nil)
+		if rc != sqlite.OK {
+			fmt.printf("Error vacuuming database: %s\n", sqlite.db_errmsg(d.db))
 			return
 		}
 
-		sqlite_data, read_err := os.read_entire_file_from_path(tmp_path, context.allocator)
-		os.remove(tmp_path)
-		if read_err != nil {
-			fmt.printf("Error reading vacuumed database: %v\n", read_err)
-			sqlite.db_close(d.db)
+		sz: i64
+		data := sqlite.serialize(d.db, "main", &sz, 0)
+		if data == nil {
+			fmt.println("Error: failed to serialize database")
 			return
 		}
+		defer sqlite.free(data)
 
+		sqlite_data := data[:sz]
 		encrypted, enc_ok := encrypt(sqlite_data, d.cfg.Keys[:])
-		delete(sqlite_data)
 		if !enc_ok {
 			fmt.println("Error: encryption failed")
-			sqlite.db_close(d.db)
 			return
 		}
 
-		data_path := data_encrypted_path(d.cfg.config_path)
+		data_path := data_path(d.cfg.config_path)
 		envr_d := envr_dir(d.cfg.config_path)
 		os.mkdir_all(envr_d)
 
@@ -128,13 +127,11 @@ db_close :: proc(d: ^Db) {
 		delete(encrypted)
 		if write_err != nil {
 			fmt.printf("Error writing encrypted database: %v\n", write_err)
-			sqlite.db_close(d.db)
 			return
 		}
 
 		d.changed = false
 	}
-	sqlite.db_close(d.db)
 }
 
 // Caller is responsible for calling:
@@ -192,22 +189,12 @@ db_list :: proc(d: ^Db, allocator := context.allocator) -> (results: [dynamic]En
 	return
 }
 
-db_vacuum_to_file :: proc(db: ^rawptr, path: string) -> bool {
-	b: strings.Builder
-	strings.builder_init(&b)
-	defer strings.builder_destroy(&b)
-	fmt.sbprintf(&b, "VACUUM INTO '%s'", path)
-	rc := sqlite.db_exec(db, to_cstring(&b), nil, nil, nil)
-	if rc != sqlite.OK {
-		fmt.printf("Error vacuuming database: %s\n", sqlite.db_errmsg(db))
-		return false
-	}
-	return true
-}
-
 db_restore_from_encrypted :: proc(db: ^rawptr, cfg: Config) -> bool {
-	data_path := data_encrypted_path(cfg.config_path)
-	encrypted_data, read_err := os.read_entire_file_from_path(data_path, context.temp_allocator)
+	encrypted_data, read_err := os.read_entire_file_from_path(
+		data_path(cfg.config_path),
+		context.allocator,
+	)
+	defer delete(encrypted_data)
 	if read_err != nil {
 		fmt.printf("Error reading encrypted database: %v\n", read_err)
 		return false
@@ -220,50 +207,32 @@ db_restore_from_encrypted :: proc(db: ^rawptr, cfg: Config) -> bool {
 	}
 	defer delete(plaintext)
 
-	tmp_path := make_temp_path()
-	write_err := os.write_entire_file(tmp_path, plaintext)
-	if write_err != nil {
-		fmt.printf("Error writing temp database: %v\n", write_err)
+	n := i64(len(plaintext))
+	buf := sqlite.malloc64(n)
+	if buf == nil {
+		fmt.println("Error: failed to allocate buffer for deserialization")
 		return false
 	}
-	defer os.remove(tmp_path)
+	copy(buf[:len(plaintext)], plaintext)
 
-	if !db_attach_and_copy(db, tmp_path) {
-		return false
-	}
-
-	return true
-}
-
-db_attach_and_copy :: proc(mem_db: ^rawptr, src_path: string) -> bool {
-	b: strings.Builder
-	strings.builder_init(&b)
-	defer strings.builder_destroy(&b)
-	fmt.sbprintf(&b, "ATTACH DATABASE '%s' AS source", src_path)
-
-	rc := sqlite.db_exec(mem_db, to_cstring(&b), nil, nil, nil)
-	if rc != sqlite.OK {
-		fmt.printf("Error attaching database: %s\n", sqlite.db_errmsg(mem_db))
-		return false
-	}
-
-	rc = sqlite.db_exec(
-		mem_db,
-		"INSERT INTO main.envr_env_files SELECT * FROM source.envr_env_files",
-		nil,
-		nil,
-		nil,
+	rc := sqlite.deserialize(
+		db,
+		"main",
+		buf,
+		n,
+		n,
+		sqlite.DESERIALIZE_FREEONCLOSE | sqlite.DESERIALIZE_RESIZEABLE,
 	)
 	if rc != sqlite.OK {
-		fmt.printf("Error copying data: %s\n", sqlite.db_errmsg(mem_db))
-		sqlite.db_exec(mem_db, "DETACH DATABASE source", nil, nil, nil)
+		sqlite.free(buf)
+		fmt.printf("Error deserializing database: %s\n", sqlite.db_errmsg(db))
 		return false
 	}
 
-	sqlite.db_exec(mem_db, "DETACH DATABASE source", nil, nil, nil)
 	return true
 }
 
+
 get_git_remotes :: proc(dir: string) -> [dynamic]string {
 	remotes: [dynamic]string
 	remote_set: map[string]bool

db_integration_test.odin

@@ -136,7 +136,7 @@ test_encrypt_write_read_decrypt :: proc(t: ^testing.T) {
 }
 
 @(test)
-test_decrypt_then_attach_sqlite :: proc(t: ^testing.T) {
+test_decrypt_then_deserialize_sqlite :: proc(t: ^testing.T) {
 	cfg := fixture_config()
 	defer {
 		delete(cfg.Keys)
@@ -164,14 +164,6 @@ test_decrypt_then_attach_sqlite :: proc(t: ^testing.T) {
 	}
 	defer delete(plaintext)
 
-	tmp_db_path := fmt.tprintf("/tmp/envr-test-attach-%d.db", os.get_pid())
-	write_err := os.write_entire_file(tmp_db_path, plaintext)
-	testing.expectf(t, write_err == nil, "failed to write temp db: %v", write_err)
-	if write_err != nil {
-		return
-	}
-	defer os.remove(tmp_db_path)
-
 	mem_db: ^rawptr
 	rc := sqlite.db_open(":memory:", &mem_db)
 	testing.expectf(t, rc == sqlite.OK, "failed to open in-memory db")
@@ -180,12 +172,25 @@ test_decrypt_then_attach_sqlite :: proc(t: ^testing.T) {
 	}
 	defer sqlite.db_close(mem_db)
 
-	create_sql: cstring = "CREATE TABLE IF NOT EXISTS envr_env_files (path TEXT PRIMARY KEY NOT NULL, remotes TEXT, sha256 TEXT NOT NULL, contents TEXT NOT NULL)"
-	rc = sqlite.db_exec(mem_db, create_sql, nil, nil, nil)
-	testing.expect(t, rc == sqlite.OK, "failed to create table")
+	n := i64(len(plaintext))
+	buf := sqlite.malloc64(n)
+	testing.expect(t, buf != nil, "malloc64 should succeed")
+	if buf == nil do return
+	copy(buf[:len(plaintext)], plaintext)
 
-	attach_ok := db_attach_and_copy(mem_db, tmp_db_path)
-	testing.expect(t, attach_ok, "failed to attach and copy")
+	rc = sqlite.deserialize(
+		mem_db,
+		"main",
+		buf,
+		n,
+		n,
+		sqlite.DESERIALIZE_FREEONCLOSE | sqlite.DESERIALIZE_RESIZEABLE,
+	)
+	testing.expect(t, rc == sqlite.OK, "deserialize should succeed")
+	if rc != sqlite.OK {
+		sqlite.free(buf)
+		return
+	}
 
 	sql: cstring = "SELECT path FROM envr_env_files"
 	stmt: ^rawptr

db_test.odin

@@ -215,7 +215,7 @@ test_db_delete_sets_changed :: proc(t: ^testing.T) {
 }
 
 @(test)
-test_db_vacuum_to_file :: proc(t: ^testing.T) {
+test_db_serialize :: proc(t: ^testing.T) {
 	d, ok := make_test_db()
 	testing.expect(t, ok, "failed to create test db")
 	if !ok do return
@@ -225,20 +225,13 @@ test_db_vacuum_to_file :: proc(t: ^testing.T) {
 	defer delete(f.Remotes)
 	db_insert(&d, f)
 
-	vacuum_path := fmt.tprintf("/tmp/envr-test-vacuum-%d.db", os.get_pid())
-	defer os.remove(vacuum_path)
+	sz: i64
+	data := sqlite.serialize(d.db, "main", &sz, 0)
+	testing.expect(t, data != nil, "serialize should return non-nil")
+	if data == nil do return
+	defer sqlite.free(data)
 
-	testing.expect(t, db_vacuum_to_file(d.db, vacuum_path), "vacuum should succeed")
-
-	info, stat_err := os.stat(vacuum_path, context.allocator)
-	defer os.file_info_delete(info, context.allocator)
-	testing.expect(t, stat_err == nil, "vacuumed file should exist")
-
-	data, read_err := os.read_entire_file_from_path(vacuum_path, context.allocator)
-	testing.expect(t, read_err == nil, "should read vacuumed file")
-	defer delete(data)
-
-	testing.expect(t, len(data) > 0, "vacuumed file should be non-empty")
+	testing.expect(t, sz > 0, "serialized size should be > 0")
 }
 
 @(test)

flake.nix

@@ -11,11 +11,12 @@
   };
 
   outputs =
-    inputs@{ flake-parts
-    , nixpkgs
-    , nixpkgs-unstable
-    , self
-    , treefmt-nix
+    inputs@{
+      flake-parts,
+      nixpkgs,
+      nixpkgs-unstable,
+      self,
+      treefmt-nix,
     }:
     flake-parts.lib.mkFlake { inherit inputs; } {
       imports = [
@@ -29,7 +30,18 @@
       ];
 
       perSystem =
-        { pkgs, system, inputs', ... }: {
+        {
+          pkgs,
+          system,
+          inputs',
+          ...
+        }:
+      let
+        mysqlite = pkgs.sqlite.overrideAttrs (old: {
+                configureFlags = (old.configureFlags or [ ]) ++ [ "--enable-deserialize" ];
+              });
+      in
+        {
           _module.args.pkgs = import nixpkgs {
             inherit system;
             config.allowUnfree = true;
@@ -64,7 +76,7 @@
 
             buildInputs = [
               pkgs.libsodium
-              pkgs.sqlite
+              mysqlite
             ];
 
             buildPhase = ''
@@ -87,7 +99,7 @@
               nushell
 
               libsodium
-              sqlite
+              mysqlite
               unstable.odin
               unstable.ols
 

sqlite/sqlite.odin

@@ -8,6 +8,9 @@ OK :: 0
 ROW :: 100
 DONE :: 101
 
+DESERIALIZE_FREEONCLOSE :: 1
+DESERIALIZE_RESIZEABLE :: 2
+
 foreign lib {
     @(link_name="sqlite3_open")
     db_open          :: proc(filename: cstring, ppDb: ^^rawptr) -> c.int ---
@@ -31,4 +34,12 @@ foreign lib {
     bind_text        :: proc(stmt: ^rawptr, idx: c.int, val: cstring, n: c.int, destructor: rawptr) -> c.int ---
     @(link_name="sqlite3_changes")
     changes          :: proc(db: ^rawptr) -> c.int ---
+    @(link_name="sqlite3_serialize")
+    serialize        :: proc(db: ^rawptr, zSchema: cstring, piSize: ^i64, mFlags: u32) -> [^]u8 ---
+    @(link_name="sqlite3_deserialize")
+    deserialize      :: proc(db: ^rawptr, zSchema: cstring, pData: [^]u8, szDb: i64, szBuf: i64, mFlags: u32) -> c.int ---
+    @(link_name="sqlite3_malloc64")
+    malloc64         :: proc(n: i64) -> [^]u8 ---
+    @(link_name="sqlite3_free")
+    free             :: proc(p: rawptr) ---
 }