yarascanner/main.go

package main

import (
	"archive/zip"
	"encoding/json"
	"errors"
	"github.com/beanstalkd/go-beanstalk"
	"github.com/hillu/go-yara"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	"github.com/spf13/afero/zipfs"
	"github.com/spf13/viper"
	"io"
	"io/fs"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path"
	"runtime"
	"strings"
	"syscall"
	"time"
)

type Job struct {
	PasteID string
	Data []byte
}

var (
	client *http.Client
)

func main() {
	viper.SetDefault("beanstalk", "127.0.0.1:11300")
	viper.SetDefault("threads", runtime.NumCPU())
	viper.SetDefault("pasteUrl", "https://paste.ee")
	viper.SetDefault("rules", "https://github.com/Neo23x0/signature-base/yara")

	viper.AutomaticEnv()

	client = &http.Client{
		Timeout: 15 * time.Second,
	}

	jobChan := make(chan Job)

	c, err := yara.NewCompiler()

	if err != nil {
		log.WithError(err).Fatal("Unable to setup new compiler")
	}

	loadRules(c)

	rules, err := c.GetRules()

	if err != nil {
		log.WithError(err).Fatal("Unable to compile rules")
	}

	threads := viper.GetInt("threads")

	log.WithField("workers", threads).Info("Starting workers")

	for i := 0; i < threads; i++ {
		go worker(rules, jobChan)
	}

	conn, err := beanstalk.Dial("tcp", viper.GetString("beanstalk"))

	if err != nil {
		log.WithError(err).Fatal("Unable to connect to beanstalkd")
	}

	go watchQueue(conn, jobChan)

	ch := make(chan os.Signal)

	signal.Notify(ch, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGINT)

	<- ch
}

func loadRules(c *yara.Compiler) {
	rulePaths := strings.Split(viper.GetString("rules"), ",")

	var err error
	var closer io.Closer

	for _, p := range rulePaths {
		var afs afero.Fs

		log.WithField("path", p).Info("Loading rules from p")

		if strings.HasPrefix(p, "http") || strings.HasPrefix(p, "git") {
			// load http rules
			afs, closer, err = loadRulesFromHttp(p)
		} else if stat, err := os.Stat(p); !os.IsNotExist(err) && stat.IsDir() {
			afs = afero.NewBasePathFs(afero.NewOsFs(), p)
		} else {
			continue
		}

		if err != nil {
			log.WithError(err).Fatalln("Unable to load rules")
		}

		afero.Walk(afs, "", func(path string, info fs.FileInfo, err error) error {
			b, err := afero.ReadFile(afs, path)

			return c.AddString(string(b), "")
		})

		if closer != nil {
			closer.Close()
		}
	}
}

func loadRulesFromHttp(ruleUrl string) (afero.Fs, io.Closer, error) {
	u, err := url.Parse(ruleUrl)

	if err != nil {
		log.WithError(err).Fatalln("Invalid URL!")
	}

	var subPath string

	if u.Host == "github.com" {
		parts := strings.Split(u.Path, "/")

		if len(parts) < 2 {
			return nil, nil, errors.New("invalid repo")
		}
		// User = parts[0], Repo = parts[1], Sub path = parts[2:]

		// https://github.com/parts[0]/parts[1]/archive/master.zip

		ruleUrl = "https://github.com/" + path.Join(parts[0], parts[1], "archive", "master.zip")
		subPath = path.Join(parts[2:]...)
	}

	res, err := client.Get(ruleUrl)

	if err != nil {
		return nil, nil, err
	}

	defer res.Body.Close()

	tmpFile, err := os.CreateTemp(os.TempDir(), "yararules.zip")

	io.Copy(tmpFile, res.Body)

	tmpFile.Seek(0, io.SeekStart)

	stat, err := tmpFile.Stat()

	if err != nil {
		return nil, nil, err
	}

	z, err := zip.NewReader(tmpFile, stat.Size())

	if err != nil {
		return nil, nil, err
	}

	return afero.NewBasePathFs(zipfs.New(z), subPath), tmpFile, nil
}

func watchQueue(c *beanstalk.Conn, jobChan chan Job) {
	for {
		id, body, err := c.Reserve(5 * time.Second)

		if err != nil {
			continue
		}

		log.WithFields(log.Fields{
			"id": id,
			"body": body,
		}).Debug("Handling job")

		var job Job

		if err = json.Unmarshal(body, &job); err != nil {
			continue
		}

		jobChan <- job
	}
}

func worker(rules *yara.Rules, jobs chan Job) {
	s, err := yara.NewScanner(rules)

	if err != nil {
		panic(err)
	}

	for {
		job := <- jobs

		processJob(s, job)
	}
}

func processJob(s *yara.Scanner, job Job) {
	matched, err := s.ScanMem(job.Data)

	if err != nil {
		return
	}

	// Respond with job
	if len(matched) < 1 {
		return
	}

	quarantine(job.PasteID, matched[0].Rule)
}

func quarantine(pasteId, reason string) {
	v := make(url.Values)

	v.Set("reason", reason)

	req, err := http.NewRequest(http.MethodPost, viper.GetString("pasteUrl") + "/admin/quarantine/" + pasteId, nil)

	if err != nil {
		return
	}

	res, err := client.Do(req)

	if err != nil {
		return
	}

	defer res.Body.Close()
}