yarascanner/main.go

package main

import (
	"encoding/json"
	"github.com/go-chi/chi"
	"github.com/hillu/go-yara/v4"
	"github.com/package-url/packageurl-go"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/viper"
	"io"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

type callbackFunc func(yara.MatchRules, error)

type Job struct {
	Data     io.ReadCloser
	Callback callbackFunc
}

var (
	client *http.Client

	jobChan = make(chan *Job)

	jobCount      int64
	detectedCount int64
)

type statsResponse struct {
	Processed int64 `json:"jobsProcessed"`
	Detected  int64 `json:"detectionCount""`
}

type response struct {
	Success      bool            `json:"success"`
	Error        error           `json:"error,omitempty"`
	MatchedRules yara.MatchRules `json:"rules,omitempty"`
}

type multiResponse struct {
	Success bool                       `json:"success"`
	Files   map[string]yara.MatchRules `json:"files,omitempty"`
}

func main() {
	viper.SetDefault("threads", runtime.NumCPU())
	viper.SetDefault("rules", "pkg:github/Neo23x0/signature-base#yara")
	viper.SetDefault("bind", ":8080")

	viper.AutomaticEnv()

	client = &http.Client{
		Timeout: 15 * time.Second,
	}

	c, err := yara.NewCompiler()

	if err != nil {
		log.WithError(err).Fatal("Unable to setup new compiler")
	}

	c.DefineVariable("filename", "")
	c.DefineVariable("filepath", "")
	c.DefineVariable("extension", "")
	c.DefineVariable("filetype", "")

	log.Info("Loading rules")

	loadRules(c)

	rules, err := c.GetRules()

	if err != nil {
		log.WithError(err).Fatal("Unable to compile rules")
	}

	threads := viper.GetInt("threads")

	log.WithField("workers", threads).Info("Starting workers")

	for i := 0; i < threads; i++ {
		go worker(rules)
	}

	r := chi.NewRouter()

	r.Get("/stats", statsHandler)
	r.Post("/scan", scanHandler)

	bind := viper.GetString("bind")

	log.WithField("bind", bind).Info("Binding to address")

	go http.ListenAndServe(bind, r)

	ch := make(chan os.Signal)

	signal.Notify(ch, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGINT)

	<-ch
}

func statsHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(statsResponse{Processed: atomic.LoadInt64(&jobCount), Detected: atomic.LoadInt64(&detectedCount)})
}

// HTTP handler for scanning files
func scanHandler(w http.ResponseWriter, r *http.Request) {
	contentType := r.Header.Get("Content-Type")

	if idx := strings.Index(contentType, ";"); idx != -1 {
		contentType = contentType[0:idx]
	}

	var res interface{}

	wg := &sync.WaitGroup{}

	switch contentType {
	case "multipart/form-data":
		if r.MultipartForm == nil {
			r.ParseMultipartForm(32 << 20)
		}

		log.WithField("contentType", contentType).Debug("Adding files from multipart form as jobs")

		fileCount := 0

		response := multiResponse{
			Success: true,
			Files:   make(map[string]yara.MatchRules),
		}

		fileLock := &sync.Mutex{}

		for _, files := range r.MultipartForm.File {
			// Append files
			for _, file := range files {
				wg.Add(1)

				fileCount++

				f, err := file.Open()

				if err != nil {
					continue
				}

				job := &Job{
					Data: f,
					Callback: func(m yara.MatchRules, err error) {
						fileLock.Lock()
						response.Files[file.Filename] = m
						fileLock.Unlock()

						wg.Done()
					},
				}

				jobChan <- job
			}
		}

		log.WithField("count", fileCount).Debug("Waiting for jobs to finish")

		wg.Wait()

		log.WithField("matches", res).Debug("Matched rules")

		res = response
	default:
		wg.Add(1)

		job := &Job{
			Data: r.Body,
			Callback: func(m yara.MatchRules, err error) {
				log.WithField("match", m).Debug("Matched rules")

				if err != nil {
					res = response{Success: false, Error: err}
				} else {
					res = response{Success: true, MatchedRules: m}
				}

				if err != nil {
					log.WithError(err).Error("Unable to send response")
				}

				wg.Done()
			},
		}

		log.WithField("contentType", contentType).Debug("Scanning contents of body")

		jobChan <- job
	}

	wg.Wait()

	w.Header().Set("Content-Type", "application/json")

	err := json.NewEncoder(w).Encode(res)

	if err != nil {
		log.WithError(err).Error("Unable to send response")
	}
}

// Load rules from the rules configuration option
func loadRules(c *yara.Compiler) {
	rulePaths := strings.Split(viper.GetString("rules"), ",")

	for _, p := range rulePaths {
		log.WithField("package", p).Info("Loading rules from package")

		instance, err := packageurl.FromString(p)

		if err != nil {
			log.WithFields(log.Fields{
				"error":   err,
				"package": p,
			}).Fatalln("Invalid rule URL")
		}

		switch instance.Type {
		case "git", "bitbucket", "github", "gitlab":
			err = loadRulesFromGit(instance, c)
		case "http", "https":
			err = loadRulesFromHttp(instance, c)
		}

		if err != nil {
			log.WithFields(log.Fields{
				"error":   err,
				"package": p,
			}).Fatalln("Unable to load rules")
		}
	}
}

// Load rules from http(s)
// TODO: Support archive files alongside standard yar files
func loadRulesFromHttp(pkg packageurl.PackageURL, c *yara.Compiler) error {
	res, err := client.Get(pkg.Type + "://" + pkg.Namespace + "/" + pkg.Name)

	if err != nil {
		return err
	}

	defer res.Body.Close()

	b, err := io.ReadAll(res.Body)

	if err != nil {
		return err
	}

	return c.AddString(string(b), "")
}

// A worker routine. Creates a new scanner instance and pulls jobs.
func worker(rules *yara.Rules) {
	s, err := yara.NewScanner(rules)

	if err != nil {
		panic(err)
	}

	for {
		job := <-jobChan

		log.Debug("Processing job")

		processJob(s, job)
	}
}

func processJob(s *yara.Scanner, job *Job) {
	m := make(yara.MatchRules, 0)

	atomic.AddInt64(&jobCount, 1)

	defer job.Data.Close()

	b, err := io.ReadAll(job.Data)

	if err != nil {
		job.Callback(nil, err)
		return
	}

	err = s.SetCallback(&m).ScanMem(b)

	if err != nil {
		job.Callback(nil, err)
		return
	}

	if len(m) > 0 {
		atomic.AddInt64(&detectedCount, 1)
	}

	job.Callback(m, nil)
}