ternfs-XTXMarkets/go/lib/scrub.go

package lib

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"
	"xtx/eggsfs/msgs"
)

type ScrubState struct {
	Migrate              MigrateStats
	CheckedBlocks        uint64
	CheckedBytes         uint64
	WorkersQueuesSize    [256]uint64
	CheckQueuesSize      [256]uint64
	DecommissionedBlocks uint64
	Cursors              [256]msgs.InodeId
	Cycles               [256]uint32
}

type ScrubOptions struct {
	NumWorkersPerShard int // how many goroutienes should be sending check request to the block services
	WorkersQueueSize   int
}

func badBlockError(err error) bool {
	return err == msgs.BAD_BLOCK_CRC || err == msgs.BLOCK_NOT_FOUND || err == msgs.BLOCK_IO_ERROR_FILE
}

func scrubFileInternal(
	log *Logger,
	client *Client,
	bufPool *BufPool,
	stats *ScrubState,
	timeStats *timeStats,
	progressReportAlert *XmonNCAlert,
	scratchFile *scratchFile,
	scrubbingMu *sync.Mutex,
	file msgs.InodeId,
) error {
	log.Debug("starting to scrub file %v", file)
	scrubbingMu.Lock() // scrubbings don't work in parallel
	defer scrubbingMu.Unlock()

	badBlock := func(blockService *msgs.BlockService, blockSize uint32, block *msgs.FetchedBlock) (bool, error) {
		err := client.CheckBlock(log, blockService, block.BlockId, blockSize, block.Crc)
		if badBlockError(err) {
			log.RaiseAlert("found bad block, block service %v, block %v: %v", blockService.Id, block.BlockId, err)
			return true, nil
		}
		return false, err
	}
	return migrateBlocksInFileGeneric(log, client, bufPool, &stats.Migrate, timeStats, progressReportAlert, "scrubbed", badBlock, scratchFile, file)
}

type scrubRequest struct {
	file         msgs.InodeId
	blockService msgs.BlockService
	block        msgs.BlockId
	size         uint32
	crc          msgs.Crc
}

func scrubWorker(
	log *Logger,
	client *Client,
	opts *ScrubOptions,
	stats *ScrubState,
	rateLimit *RateLimit,
	shid msgs.ShardId,
	workerChan chan *scrubRequest,
	terminateChan chan any,
	scratchFile *scratchFile,
	scrubbingMu *sync.Mutex,
) {
	bufPool := NewBufPool()
	blockNotFoundAlert := log.NewNCAlert(0)
	defer log.ClearNC(blockNotFoundAlert)
	for {
		req := <-workerChan
		if req == nil {
			log.Debug("worker for shard %v terminating", shid)
			workerChan <- nil // terminate the other workers, too
			log.Debug("worker for shard %v terminated", shid)
			return
		}
		if rateLimit != nil {
			rateLimit.Acquire()
		}
		atomic.StoreUint64(&stats.WorkersQueuesSize[shid], uint64(len(workerChan)))
		canIgnoreError := req.blockService.Flags.HasAny(msgs.EGGSFS_BLOCK_SERVICE_STALE | msgs.EGGSFS_BLOCK_SERVICE_DECOMMISSIONED | msgs.EGGSFS_BLOCK_SERVICE_NO_WRITE)
		if req.blockService.Flags.HasAny(msgs.EGGSFS_BLOCK_SERVICE_DECOMMISSIONED) {
			atomic.AddUint64(&stats.DecommissionedBlocks, 1)
		}
		err := client.CheckBlock(log, &req.blockService, req.block, req.size, req.crc)
		if badBlockError(err) {
			atomic.AddUint64(&stats.CheckedBlocks, 1)
			if err == msgs.BAD_BLOCK_CRC {
				atomic.AddUint64(&stats.CheckedBytes, uint64(req.size))
			}
			for attempts := 1; ; attempts++ {
				if err := scrubFileInternal(log, client, bufPool, stats, nil, nil, scratchFile, scrubbingMu, req.file); err != nil {
					if err == msgs.BLOCK_NOT_FOUND {
						log.RaiseNC(blockNotFoundAlert, "could not migrate blocks in file %v after %v attempts because a block was not found in it. this is probably due to conflicts with other migrations or scrubbing. will retry in one second.", req.file, attempts)
						time.Sleep(time.Second)
					} else {
						log.Info("could not scrub file %v, will terminate: %v", req.file, err)
						select {
						case terminateChan <- err:
						default:
						}
						return
					}
				} else {
					log.ClearNC(blockNotFoundAlert)
					break
				}
			}
		} else if err == msgs.BLOCK_IO_ERROR_DEVICE {
			// This is almost certainly a broken server. We want migration to take
			// care of this -- otherwise the scrubber will be stuck on tons of
			// these errors.
			log.Info("got IO error for file %v (block %v, block service %v), ignoring: %v", req.file, req.block, req.blockService.Id, err)
		} else if err != nil {
			if canIgnoreError {
				log.Debug("could not check block %v in file %v block service %v, but can ignore error (block service is probably decommissioned): %v", req.block, req.file, req.blockService.Id, err)
			} else {
				log.Info("could not check block %v for file %v, will terminate: %v", req.block, req.file, err)
				select {
				case terminateChan <- err:
				default:
				}
				return
			}
		} else {
			atomic.AddUint64(&stats.CheckedBlocks, 1)
			atomic.AddUint64(&stats.CheckedBytes, uint64(req.size))
		}
	}
}

func scrubScraper(
	log *Logger,
	client *Client,
	stats *ScrubState,
	shid msgs.ShardId,
	terminateChan chan any,
	workerChan chan *scrubRequest,
) {
	fileReq := &msgs.VisitFilesReq{BeginId: stats.Cursors[shid]}
	fileResp := &msgs.VisitFilesResp{}
	for {
		if err := client.ShardRequest(log, msgs.ShardId(shid), fileReq, fileResp); err != nil {
			log.Info("could not get files: %v", err)
			select {
			case terminateChan <- err:
			default:
			}
			return
		}
		log.Debug("will migrate %d files", len(fileResp.Ids))
		for _, file := range fileResp.Ids {
			spansReq := msgs.FileSpansReq{FileId: file}
			spansResp := msgs.FileSpansResp{}
			for {
				if err := client.ShardRequest(log, file.Shard(), &spansReq, &spansResp); err != nil {
					log.Info("could not get spans: %v", err)
					select {
					case terminateChan <- err:
					default:
					}
					return
				}
				for _, span := range spansResp.Spans {
					if span.Header.StorageClass == msgs.INLINE_STORAGE {
						continue
					}
					body := span.Body.(*msgs.FetchedBlocksSpan)
					for _, block := range body.Blocks {
						size := body.CellSize * uint32(body.Stripes)
						blockService := spansResp.BlockServices[block.BlockServiceIx]
						workerChan <- &scrubRequest{
							file:         file,
							blockService: blockService,
							block:        block.BlockId,
							size:         size,
							crc:          block.Crc,
						}
					}
				}
				spansReq.ByteOffset = spansResp.NextOffset
				if spansReq.ByteOffset == 0 {
					break
				}
			}
		}
		stats.Cursors[shid] = fileResp.NextId
		fileReq.BeginId = fileResp.NextId
		if fileReq.BeginId == 0 {
			// this will terminate all the workers
			log.Debug("file scraping done for shard %v, terminating workers", shid)
			atomic.AddUint32(&stats.Cycles[shid], 1)
			workerChan <- nil
			return
		}
	}
}

func ScrubFile(
	log *Logger,
	client *Client,
	stats *ScrubState,
	file msgs.InodeId,
) error {
	bufPool := NewBufPool()
	scratchFile := scratchFile{}
	keepAlive := startToKeepScratchFileAlive(log, client, &scratchFile)
	defer keepAlive.stop()
	var scrubbingMu sync.Mutex
	return scrubFileInternal(log, client, bufPool, stats, nil, nil, &scratchFile, &scrubbingMu, file)
}

func ScrubFiles(
	log *Logger,
	client *Client,
	opts *ScrubOptions,
	rateLimit *RateLimit,
	stats *ScrubState,
	shid msgs.ShardId,
) error {
	if opts.NumWorkersPerShard <= 0 {
		panic(fmt.Errorf("the number of senders should be positive, got %v", opts.NumWorkersPerShard))
	}
	log.Info("starting to scrub files for shard %v", shid)
	terminateChan := make(chan any, 1)
	sendChan := make(chan *scrubRequest, opts.WorkersQueueSize)
	scratchFile := &scratchFile{}
	keepAlive := startToKeepScratchFileAlive(log, client, scratchFile)
	defer func() {
		keepAlive.stop()
	}()

	go func() {
		defer func() { HandleRecoverChan(log, terminateChan, recover()) }()
		scrubScraper(log, client, stats, shid, terminateChan, sendChan)
	}()

	var workersWg sync.WaitGroup
	workersWg.Add(opts.NumWorkersPerShard)
	var scrubbingMu sync.Mutex
	for i := 0; i < opts.NumWorkersPerShard; i++ {
		go func() {
			defer func() { HandleRecoverChan(log, terminateChan, recover()) }()
			scrubWorker(log, client, opts, stats, rateLimit, shid, sendChan, terminateChan, scratchFile, &scrubbingMu)
			workersWg.Done()
		}()
	}
	go func() {
		workersWg.Wait()
		log.Info("all workers terminated for shard %v, we're done", shid)
		select {
		case terminateChan <- nil:
		default:
		}
	}()

	err := <-terminateChan
	if err == nil {
		return nil
	} else {
		log.Info("could not scrub files: %v", err)
		return err.(error)
	}
}

func ScrubFilesInAllShards(
	log *Logger,
	client *Client,
	opts *ScrubOptions,
	rateLimit *RateLimit,
	state *ScrubState,
) error {
	terminateChan := make(chan any, 1)

	var wg sync.WaitGroup
	wg.Add(256)
	for i := 0; i < 256; i++ {
		shid := msgs.ShardId(i)
		go func() {
			defer func() { HandleRecoverChan(log, terminateChan, recover()) }()
			if err := ScrubFiles(log, client, opts, rateLimit, state, shid); err != nil {
				panic(err)
			}
			wg.Done()
		}()
	}
	go func() {
		wg.Wait()
		log.Info("scrubbing terminated in all shards")
		terminateChan <- nil
	}()

	err := <-terminateChan
	if err == nil {
		return nil
	} else {
		log.Info("could not scrub files: %v", err)
		return err.(error)
	}
}