gitrob/main.go

package main

import (
  "fmt"
  "os"
  "strings"
  "sync"
  "time"

  "github.com/michenriksen/gitrob/core"
)

var (
  sess *core.Session
  err  error
)

func GatherTargets(sess *core.Session) {
  sess.Stats.Status = core.StatusGathering
  sess.Out.Important("Gathering targets...\n")
  for _, login := range sess.Options.Logins {
    target, err := core.GetUserOrOrganization(login, sess.GithubClient)
    if err != nil {
      sess.Out.Error(" Error retrieving information on %s: %s\n", login, err)
      continue
    }
    sess.Out.Debug("%s (ID: %d) type: %s\n", *target.Login, *target.ID, *target.Type)
    sess.AddTarget(target)
    if *sess.Options.NoExpandOrgs == false && *target.Type == "Organization" {
      sess.Out.Debug("Gathering members of %s (ID: %d)...\n", *target.Login, *target.ID)
      members, err := core.GetOrganizationMembers(target.Login, sess.GithubClient)
      if err != nil {
        sess.Out.Error(" Error retrieving members of %s: %s\n", *target.Login, err)
        continue
      }
      for _, member := range members {
        sess.Out.Debug("Adding organization member %s (ID: %d) to targets\n", *member.Login, *member.ID)
        sess.AddTarget(member)
      }
    }
  }
}

func GatherRepositories(sess *core.Session) {
  var ch = make(chan *core.GithubOwner, len(sess.Targets))
  var wg sync.WaitGroup
  var threadNum int
  if len(sess.Targets) == 1 {
    threadNum = 1
  } else if len(sess.Targets) <= *sess.Options.Threads {
    threadNum = len(sess.Targets) - 1
  } else {
    threadNum = *sess.Options.Threads
  }
  wg.Add(threadNum)
  sess.Out.Debug("Threads for repository gathering: %d\n", threadNum)
  for i := 0; i < threadNum; i++ {
    go func() {
      for {
        target, ok := <-ch
        if !ok {
          wg.Done()
          return
        }
        repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient)
        if err != nil {
          sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err)
        }
        if len(repos) == 0 {
          continue
        }
        for _, repo := range repos {
          sess.Out.Debug(" Retrieved repository: %s\n", *repo.FullName)
          sess.AddRepository(repo)
        }
        sess.Stats.IncrementTargets()
        sess.Out.Info(" Retrieved %d %s from %s\n", len(repos), core.Pluralize(len(repos), "repository", "repositories"), *target.Login)
      }
    }()
  }

  for _, target := range sess.Targets {
    ch <- target
  }
  close(ch)
  wg.Wait()
}

func AnalyzeRepositories(sess *core.Session) {
  sess.Stats.Status = core.StatusAnalyzing
  var ch = make(chan *core.GithubRepository, len(sess.Repositories))
  var wg sync.WaitGroup
  var threadNum int
  if len(sess.Repositories) == 1 {
    threadNum = 1
  } else if len(sess.Repositories) <= *sess.Options.Threads {
    threadNum = len(sess.Repositories) - 1
  } else {
    threadNum = *sess.Options.Threads
  }
  wg.Add(threadNum)
  sess.Out.Debug("Threads for repository analysis: %d\n", threadNum)

  sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories"))

  for i := 0; i < threadNum; i++ {
    go func(tid int) {
      for {
        sess.Out.Debug("[THREAD #%d] Requesting new repository to analyze...\n", tid)
        repo, ok := <-ch
        if !ok {
          sess.Out.Debug("[THREAD #%d] No more tasks, marking WaitGroup as done\n", tid)
          wg.Done()
          return
        }

        sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName)
        clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, *sess.Options.CommitDepth)
        if err != nil {
          if err.Error() != "remote repository is empty" {
            sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err)
          }
          sess.Stats.IncrementRepositories()
          sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories))
          continue
        }
        sess.Out.Debug("[THREAD #%d][%s] Cloned repository to: %s\n", tid, *repo.FullName, path)

        history, err := core.GetRepositoryHistory(clone)
        if err != nil {
          sess.Out.Error("[THREAD #%d][%s] Error getting commit history: %s\n", tid, *repo.FullName, err)
          os.RemoveAll(path)
          sess.Stats.IncrementRepositories()
          sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories))
          continue
        }
        sess.Out.Debug("[THREAD #%d][%s] Number of commits: %d\n", tid, *repo.FullName, len(history))

        for _, commit := range history {
          sess.Out.Debug("[THREAD #%d][%s] Analyzing commit: %s\n", tid, *repo.FullName, commit.Hash)
          changes, _ := core.GetChanges(commit, clone)
          sess.Out.Debug("[THREAD #%d][%s] Changes in %s: %d\n", tid, *repo.FullName, commit.Hash, len(changes))
          for _, change := range changes {
            changeAction := core.GetChangeAction(change)
            path := core.GetChangePath(change)
            matchFile := core.NewMatchFile(path)
            sess.Out.Debug("[THREAD #%d][%s] Matching: %s...\n", tid, *repo.FullName, matchFile.Path)
            for _, signature := range core.Signatures {
              if signature.Match(matchFile) {

                finding := &core.Finding{
                  FilePath:        path,
                  Action:          changeAction,
                  Description:     signature.Description(),
                  Comment:         signature.Comment(),
                  RepositoryOwner: *repo.Owner,
                  RepositoryName:  *repo.Name,
                  CommitHash:      commit.Hash.String(),
                  CommitMessage:   strings.TrimSpace(commit.Message),
                  CommitAuthor:    commit.Author.String(),
                }
                finding.Initialize()
                sess.AddFinding(finding)

                sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description)
                sess.Out.Info("  Path.......: %s\n", finding.FilePath)
                sess.Out.Info("  Repo.......: %s\n", *repo.FullName)
                sess.Out.Info("  Message....: %s\n", core.TruncateString(finding.CommitMessage, 100))
                sess.Out.Info("  Author.....: %s\n", finding.CommitAuthor)
                if finding.Comment != "" {
                  sess.Out.Info("  Comment....: %s\n", finding.Comment)
                }
                sess.Out.Info("  File URL...: %s\n", finding.FileUrl)
                sess.Out.Info("  Commit URL.: %s\n", finding.CommitUrl)
                sess.Out.Info(" ------------------------------------------------\n\n")
                sess.Stats.IncrementFindings()
                break
              }
            }
            sess.Stats.IncrementFiles()
          }
          sess.Stats.IncrementCommits()
          sess.Out.Debug("[THREAD #%d][%s] Done analyzing changes in %s\n", tid, *repo.FullName, commit.Hash)
        }
        sess.Out.Debug("[THREAD #%d][%s] Done analyzing commits\n", tid, *repo.FullName)
        os.RemoveAll(path)
        sess.Out.Debug("[THREAD #%d][%s] Deleted %s\n", tid, *repo.FullName, path)
        sess.Stats.IncrementRepositories()
        sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories))
      }
    }(i)
  }
  for _, repo := range sess.Repositories {
    ch <- repo
  }
  close(ch)
  wg.Wait()
}

func PrintSessionStats(sess *core.Session) {
  sess.Out.Info("\nFindings....: %d\n", sess.Stats.Findings)
  sess.Out.Info("Files.......: %d\n", sess.Stats.Files)
  sess.Out.Info("Commits.....: %d\n", sess.Stats.Commits)
  sess.Out.Info("Repositories: %d\n", sess.Stats.Repositories)
  sess.Out.Info("Targets.....: %d\n\n", sess.Stats.Targets)
}

func main() {
  if sess, err = core.NewSession(); err != nil {
    fmt.Println(err)
    os.Exit(1)
  }

  sess.Out.Info("%s\n\n", core.ASCIIBanner)
  sess.Out.Important("%s v%s started at %s\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339))
  sess.Out.Important("Loaded %d signatures\n", len(core.Signatures))
  sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port)

  if sess.Stats.Status == "finished" {
    sess.Out.Important("Loaded session file: %s\n", *sess.Options.Load)
  } else {
    if len(sess.Options.Logins) == 0 {
      sess.Out.Fatal("Please provide at least one GitHub organization or user\n")
    }

    GatherTargets(sess)
    GatherRepositories(sess)
    AnalyzeRepositories(sess)
    sess.Finish()

    if *sess.Options.Save != "" {
      err := sess.SaveToFile(*sess.Options.Save)
      if err != nil {
        sess.Out.Error("Error saving session to %s: %s\n", *sess.Options.Save, err)
      }
      sess.Out.Important("Saved session to: %s\n\n", *sess.Options.Save)
    }
  }

  PrintSessionStats(sess)
  sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n")
  select {}
}