Delete old Go files

andybalholm
Mar 22, 2023, 4:22 PM
P6D65XL7ENXQO32YQSG6I655EBBUKF2XSIQTVCLPEWFR4S5KLPVAC

Dependencies

  • [2] RGK2IUMO If email address is not in log, get it from pijul change
  • [3] R7WB2ZZZ Import and export marks
  • [4] Y7VFVY6E Initial dummy version
  • [5] K23EJ6EJ Process commits in reverse
  • [6] YANQZYFX
  • [7] NXJB3UTC Deduplicate blobs
  • [8] RFMRCLJX Remove empty parentheses after author name
  • [9] 5ETDKF5F Start on data structures to represent fast-export stream
  • [10] 534I6MRX Add --channel flag
  • [11] OLMQ7EVU Add --branch flag
  • [12] IYGPU3IZ finish rewriting in rust
  • [13] DKHKVC66 Add a README
  • [14] RTQQLOCO Use real metadata, but no content yet
  • [15] YBAXM44P Refactor marks
  • [16] KZ4XMKSP Make a temporary clone of the repository
  • [17] P2B4ZSO5 Include file content
  • [18] EGKSUIOB Better handling of missing email address
  • [19] 2J4YY37D Use "raw" date format
  • [20] TQBJZLD7 RIIR: hello, world

Change contents

  • file deletion: main.go (----------)
    [4.1][4.0:31](),[4.31][4.32:32]()
    package main
    import (
    "fmt"
    "time"
    )
    func formatTime(t time.Time) string {
    return fmt.Sprintf("%d %s", t.Unix(), t.Format("-0700"))
    }
    type change struct {
    Hash string `json:"hash"`
    Authors []string `json:"authors"`
    Timestamp time.Time `json:"timestamp"`
    Message string `json:"message"`
    Description string `json:"description"`
    mark int
    exported bool
    }
    func main() {
    flag.Parse()
    // Make a clone of the repository to work with, so that we don't mess up the original.
    tempDir, err := os.MkdirTemp("", "")
    if err != nil {
    printErrorAndExit("Error creating temporary directory:", err)
    }
    tempRepo := filepath.Join(tempDir, "repo")
    _, err = exec.Command("pijul", "clone", "--channel", *channel, *repo, tempRepo).Output()
    if err != nil {
    printErrorAndExit("Error cloning the repository:", err)
    }
    if err := os.Chdir(tempRepo); err != nil {
    printErrorAndExit("Error changing to the clone of the repository:", err)
    }
    logBytes, err := exec.Command("pijul", "log", "--description", "--output-format=json").Output()
    if err != nil {
    printErrorAndExit("Error running pijul log:", err)
    }
    var changes []change
    err = json.Unmarshal(logBytes, &changes)
    if err != nil {
    printErrorAndExit("Error parsing pijul log output:", err)
    }
    // If the first change is empty, skip it.
    if len(changes[len(changes)-1].Authors) == 0 {
    changes = changes[:len(changes)-1]
    }
    stream := new(FastExportStream)
    if *markFile != "" {
    if err := stream.marks.Import(*markFile); err != nil {
    printErrorAndExit("Error loading marks file:", err)
    }
    }
    stream.marks.MarkChanges(changes)
    for changeIndex, c := range changes {
    if c.exported {
    break
    }
    if changeIndex > 0 {
    if _, err := exec.Command("pijul", "unrecord", "--reset", changes[changeIndex-1].Hash).Output(); err != nil {
    printErrorAndExit("Error unrecording change "+changes[changeIndex-1].Hash+":", err)
    }
    }
    var commit Commit
    commit.Mark = c.mark
    author := c.Authors[0]
    author = strings.Replace(author, "() <", "<", 1)
    if !strings.Contains(author, "<") {
    author += " <>"
    // Since the email address is missing from the log,
    // we'll try to find it in the output of pijul change.
    changeInfo, err := exec.Command("pijul", "change", c.Hash).Output()
    if err != nil {
    printErrorAndExit("Error from pijul change:", err)
    }
    if i := bytes.Index(changeInfo, authorsHeader); i != -1 {
    var name, email string
    s := bufio.NewScanner(bytes.NewReader(changeInfo[i+len(authorsHeader):]))
    for s.Scan() {
    line := s.Text()
    if line == "" {
    break
    }
    if strings.HasPrefix(line, "email =") {
    email = unquote(strings.TrimPrefix(line, "email ="))
    } else if strings.HasPrefix(line, "name =") {
    name = unquote(strings.TrimPrefix(line, "name ="))
    }
    }
    if name != "" && email != "" {
    author = name + " <" + email + ">"
    }
    }
    }
    commit.Committer = author
    commit.Timestamp = c.Timestamp
    commit.Branch = *branch
    message := c.Message
    if c.Description != "" {
    message += "\n\n" + c.Description
    }
    commit.Message = message
    if changeIndex < len(changes)-1 {
    commit.From = changes[changeIndex+1].mark
    }
    // To specify the content for the commit, we remove everything
    // and then add back in all the files that are present after the change.
    commit.DeleteAll = true
    listing, err := exec.Command("pijul", "list").Output()
    if err != nil {
    printErrorAndExit("Error from pijul list:", err)
    }
    for _, f := range strings.Split(string(listing), "\n") {
    if f == "" {
    continue
    }
    info, err := os.Stat(f)
    if err != nil {
    printErrorAndExit("Error from Stat:", err)
    }
    if info.IsDir() {
    continue
    }
    data, err := os.ReadFile(f)
    if err != nil {
    printErrorAndExit("Error reading "+f+":", err)
    }
    b := stream.AddBlob(data)
    commit.Modifications = append(commit.Modifications, FileModify{Blob: b, Path: f})
    }
    stream.AddCommit(commit)
    }
    stream.ReverseCommits()
    if err := stream.WriteTo(os.Stdout); err != nil {
    printErrorAndExit("Error writing output stream:", err)
    }
    if err := os.RemoveAll(tempDir); err != nil {
    printErrorAndExit("Error removing temporary directory:", err)
    }
    if *markFile != "" {
    if err := stream.marks.Export(*markFile); err != nil {
    printErrorAndExit("Error writing marks file:", err)
    }
    }
    }
    if *branch == "" {
    *branch = *channel
    }
    func printErrorAndExit(description string, err error) {
    msg := err.Error()
    if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 {
    msg = string(err.Stderr)
    }
    fmt.Fprintln(os.Stderr, description, msg)
    os.Exit(2)
    }
    // unquote tries to unquote various types of quoted strings, and returns the
    // result. If none of the formats works, it returns s unchanged except for
    // trimming off whitespace.
    func unquote(s string) string {
    s = strings.TrimSpace(s)
    switch {
    case strings.HasPrefix(s, "'''") && strings.HasSuffix(s, "'''"):
    return strings.TrimPrefix(strings.TrimSuffix(s, "'''"), "'''")
    case strings.HasPrefix(s, "'") && strings.HasSuffix(s, "'"):
    return strings.TrimPrefix(strings.TrimSuffix(s, "'"), "'")
    case strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\""):
    unquoted, err := strconv.Unquote(s)
    if err == nil {
    return unquoted
    }
    }
    return s
    }
    var authorsHeader = []byte("\n[[authors]]\n")
    var (
    repo = flag.String("repo", ".", "path of the repository to export")
    channel = flag.String("channel", "main", "which channel to export")
    branch = flag.String("branch", "", "destination branch in Git (default is the same as channel)")
    markFile = flag.String("marks", "", "path to file to store persistent marks")
    )
    "os"
    "os/exec"
    "path/filepath"
    "strconv"
    "strings"
    "encoding/json"
    "flag"
    "bufio"
    "bytes"
  • file deletion: marks.go (----------)
    [4.1][4.0:32](),[4.32][4.33:33]()
    package main
    type Marks struct {
    marks int
    changes []markedChange
    }
    func (m *Marks) Next() int {
    m.marks++
    return m.marks
    }
    func (m *Marks) MarkChanges(changes []change) {
    for i := range changes {
    if mark, ok := hashToMark[changes[i].Hash]; ok {
    changes[i].mark = mark
    changes[i].exported = true
    } else {
    changes[i].mark = m.Next()
    m.changes = append(m.changes, markedChange{
    mark: changes[i].mark,
    hash: changes[i].Hash,
    })
    }
    }
    }
    // Import loads a marks file analogous to those used by git fast-export's
    // --import-marks and --export-marks switches.
    func (m *Marks) Import(filename string) error {
    f, err := os.Open(filename)
    if err != nil {
    return err
    }
    defer f.Close()
    br := bufio.NewReader(f)
    for {
    var mc markedChange
    n, err := fmt.Fscanf(br, ":%d %s\n", &mc.mark, &mc.hash)
    if err == io.ErrUnexpectedEOF && n == 0 {
    return nil
    }
    if err != nil {
    return err
    }
    m.changes = append(m.changes, mc)
    }
    }
    func (m *Marks) Export(filename string) error {
    f, err := os.Create(filename)
    if err != nil {
    return err
    }
    for _, c := range m.changes {
    fmt.Fprintf(f, ":%d %s\n", c.mark, c.hash)
    }
    return f.Close()
    }
    // Make a lookup table for existing marks, and make sure we don't
    // reuse them.
    hashToMark := make(map[string]int)
    for _, mc := range m.changes {
    hashToMark[mc.hash] = mc.mark
    if mc.mark > m.marks {
    m.marks = mc.mark
    }
    }
    import (
    "bufio"
    "fmt"
    "io"
    "os"
    )
    type markedChange struct {
    mark int
    hash string
    }
  • file deletion: fast-export.go (----------)
    [4.1][4.242:280](),[4.280][4.281:281]()
    package main
    import (
    "fmt"
    "io"
    "time"
    )
    type FileModify struct {
    Blob int
    Path string
    }
    func (f FileModify) WriteTo(w io.Writer) error {
    _, err := fmt.Fprintf(w, "M 644 :%d %s\n", f.Blob, f.Path)
    return err
    }
    type Commit struct {
    Mark int
    Committer string
    Timestamp time.Time
    Message string
    DeleteAll bool
    Modifications []FileModify
    }
    func (c Commit) WriteTo(w io.Writer) error {
    if _, err := fmt.Fprintln(w, "commit refs/heads/"+c.Branch); err != nil {
    return err
    }
    if c.Mark != 0 {
    if _, err := fmt.Fprintf(w, "mark :%d\n", c.Mark); err != nil {
    return err
    }
    }
    if _, err := fmt.Fprintln(w, "committer", c.Committer, formatTime(c.Timestamp)); err != nil {
    return err
    }
    if _, err := fmt.Fprintln(w, "data", len(c.Message)); err != nil {
    return err
    }
    if _, err := fmt.Fprintln(w, c.Message); err != nil {
    return err
    }
    if c.DeleteAll {
    if _, err := fmt.Fprintln(w, "deleteall"); err != nil {
    return err
    }
    }
    for _, m := range c.Modifications {
    if err := m.WriteTo(w); err != nil {
    return err
    }
    if c.From != 0 {
    if _, err := fmt.Fprintf(w, "from :%d\n", c.From); err != nil {
    return err
    }
    }
    }
    if _, err := fmt.Fprintln(w); err != nil {
    return err
    }
    return nil
    }
    // AddBlob adds a blob to the stream and returns its mark.
    func (f *FastExportStream) AddBlob(data []byte) int {
    hash := sha512.Sum512(data)
    if mark, ok := f.blobIndex[hash]; ok {
    return mark
    }
    b := Blob{
    Mark: f.marks.Next(),
    Data: data,
    }
    f.Blobs = append(f.Blobs, b)
    return b.Mark
    if f.blobIndex == nil {
    f.blobIndex = make(map[[64]byte]int)
    }
    f.blobIndex[hash] = b.Mark
    }
    }
    func (f *FastExportStream) WriteTo(w io.Writer) error {
    for _, b := range f.Blobs {
    if err := b.WriteTo(w); err != nil {
    return err
    }
    }
    for _, c := range f.Commits {
    if err := c.WriteTo(w); err != nil {
    return err
    }
    }
    }
    return nil
    }
    type Blob struct {
    Mark int
    Data []byte
    }
    func (b Blob) WriteTo(w io.Writer) error {
    if _, err := fmt.Fprintln(w, "blob"); err != nil {
    return err
    }
    if b.Mark != 0 {
    if _, err := fmt.Fprintf(w, "mark :%d\n", b.Mark); err != nil {
    return err
    }
    }
    if _, err := fmt.Fprintln(w, "data", len(b.Data)); err != nil {
    return err
    }
    if _, err := w.Write(b.Data); err != nil {
    return err
    }
    if _, err := fmt.Fprintln(w); err != nil {
    return err
    }
    return nil
    }
    // A FastExportStream is an in-memory representation of a fast-export stream.
    type FastExportStream struct {
    Commits []Commit
    Blobs []Blob
    marks Marks
    blobIndex map[[64]byte]int
    }
    func (f *FastExportStream) AddCommit(c Commit) {
    if c.Mark == 0 {
    c.Mark = f.marks.Next()
    }
    f.Commits = append(f.Commits, c)
    }
    func (f *FastExportStream) ReverseCommits() {
    for i, j := 0, len(f.Commits)-1; j > i; i, j = i+1, j-1 {
    f.Commits[i], f.Commits[j] = f.Commits[j], f.Commits[i]
    From int
    Branch string
    "crypto/sha512"
  • file deletion: go.mod (----------)
    [4.1][4.387:417](),[4.417][4.418:418]()
    module pijul-export
    go 1.20