Include meta in ReadAllNeedles (#3991)

This is useful for doing backups on the data so we can accurately store the
last modified time, the compression state, and verify the crc.

Previously we were doing VolumeNeedleStatus and then an HTTP request which
needlessly read from the dat file twice.
This commit is contained in:
James Hartig 2022-11-20 23:19:41 -05:00 committed by GitHub
parent 70a4c98b00
commit 4c85da7844
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 746 additions and 708 deletions

View file

@ -5,13 +5,14 @@ import (
"errors"
"flag"
"fmt"
"io"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"io"
)
var (
@ -29,7 +30,7 @@ func main() {
vid := uint32(*volumeId)
eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob))
fmt.Printf("%d,%x%08x %d %v %d %x\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob), resp.NeedleBlobCompressed, resp.LastModified, resp.Crc)
return nil
}

View file

@ -317,6 +317,9 @@ message ReadAllNeedlesResponse {
uint64 needle_id = 2;
uint32 cookie = 3;
bytes needle_blob = 5;
bool needle_blob_compressed = 6;
uint64 last_modified = 7;
uint32 crc = 8;
}
message VolumeTailSenderRequest {

File diff suppressed because it is too large Load diff

View file

@ -30,10 +30,13 @@ func (scanner *VolumeFileScanner4ReadAll) VisitNeedle(n *needle.Needle, offset i
}
sendErr := scanner.Stream.Send(&volume_server_pb.ReadAllNeedlesResponse{
VolumeId: uint32(scanner.V.Id),
NeedleId: uint64(n.Id),
Cookie: uint32(n.Cookie),
NeedleBlob: n.Data,
VolumeId: uint32(scanner.V.Id),
NeedleId: uint64(n.Id),
Cookie: uint32(n.Cookie),
NeedleBlob: n.Data,
NeedleBlobCompressed: n.IsCompressed(),
LastModified: n.LastModified,
Crc: n.Checksum.Value(),
})
if sendErr != nil {
return sendErr