src/api2/admin/datastore/backup.rs: verify chunk offset

This clearly shows that the current approach does not work - seems we get
chunks out of order.

src/api2/admin/datastore/backup.rs

@@ -230,13 +230,19 @@ pub fn api_method_dynamic_append() -> ApiMethod {
     ApiMethod::new(
         dynamic_append,
         ObjectSchema::new("Append chunk to dynamic index writer.")
+            .required("wid", IntegerSchema::new("Dynamic writer ID.")
+                      .minimum(1)
+                      .maximum(256)
+            )
             .required("digest-list", ArraySchema::new(
                 "Chunk digest list.",
                 StringSchema::new("Chunk digest.").into())
             )
-            .required("wid", IntegerSchema::new("Dynamic writer ID.")
-                      .minimum(1)
-                      .maximum(256)
+            .required("offset-list", ArraySchema::new(
+                "Chunk offset list.",
+                IntegerSchema::new("Corresponding chunk offsets.")
+                    .minimum(0)
+                    .into())
             )
     )
 }
@@ -249,17 +255,23 @@ fn dynamic_append (
 
     let wid = tools::required_integer_param(&param, "wid")? as usize;
     let digest_list = tools::required_array_param(&param, "digest-list")?;
+    let offset_list = tools::required_array_param(&param, "offset-list")?;
 
     println!("DIGEST LIST LEN {}", digest_list.len());
 
+    if offset_list.len() != digest_list.len() {
+        bail!("offset list has wrong length ({} != {})", offset_list.len(), digest_list.len());
+    }
+
     let env: &BackupEnvironment = rpcenv.as_ref();
 
-    for item in digest_list {
+    for (i, item) in digest_list.iter().enumerate() {
         let digest_str = item.as_str().unwrap();
         let digest = crate::tools::hex_to_digest(digest_str)?;
+        let offset = offset_list[i].as_u64().unwrap();
         let size = env.lookup_chunk(&digest).ok_or_else(|| format_err!("no such chunk {}", digest_str))?;
 
-        env.dynamic_writer_append_chunk(wid, size, &digest)?;
+        env.dynamic_writer_append_chunk(wid, offset, size, &digest)?;
 
         env.log(format!("sucessfully added chunk {} to dynamic index {}", digest_str, wid));
     }

src/api2/admin/datastore/backup/environment.rs

@@ -123,7 +123,7 @@ impl BackupEnvironment {
     }
 
     /// Append chunk to dynamic writer
-    pub fn dynamic_writer_append_chunk(&self, wid: usize, size: u32, digest: &[u8; 32]) -> Result<(), Error> {
+    pub fn dynamic_writer_append_chunk(&self, wid: usize, offset: u64, size: u32, digest: &[u8; 32]) -> Result<(), Error> {
         let mut state = self.state.lock().unwrap();
 
         state.ensure_unfinished()?;
@@ -136,6 +136,11 @@ impl BackupEnvironment {
         data.offset += size as u64;
         data.chunk_count += 1;
 
+        if data.offset != offset {
+            bail!("dynamic writer '{}' append chunk failed - got strange chunk offset ({} != {})",
+                  data.name, data.offset, offset);
+        }
+
         data.index.add_chunk(data.offset, digest)?;
 
         Ok(())

src/api2/admin/datastore/backup/upload_chunk.rs

@@ -62,6 +62,9 @@ pub fn api_method_upload_dynamic_chunk() -> ApiAsyncMethod {
                       .minimum(1)
                       .maximum(256)
             )
+            .required("offset", IntegerSchema::new("Chunk offset (end of chunk).")
+                      .minimum(0)
+            )
             .required("size", IntegerSchema::new("Chunk size.")
                       .minimum(1)
                       .maximum(1024*1024*16)
@@ -78,8 +81,10 @@ fn upload_dynamic_chunk(
 ) -> Result<BoxFut, Error> {
 
     let size = tools::required_integer_param(&param, "size")? as u32;
+    let offset = tools::required_integer_param(&param, "offset")? as u64;
     let wid = tools::required_integer_param(&param, "wid")? as usize;
 
+    println!("upload_dynamic_chunk: {} bytes, offset {}", size, offset);
 
     let env: &BackupEnvironment = rpcenv.as_ref();
 
@@ -91,7 +96,7 @@ fn upload_dynamic_chunk(
 
             let result = result.and_then(|(digest, size)| {
                 env.register_chunk(digest, size)?;
-                env.dynamic_writer_append_chunk(wid, size, &digest)?;
+                env.dynamic_writer_append_chunk(wid, offset, size, &digest)?;
                 Ok(json!(tools::digest_to_hex(&digest)))
             });