Add line counting mode, add rate limit, add static build (#25)

Author: vearutop

.github/workflows/release-assets.yml

@@ -24,7 +24,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Build artifacts
         run: |
-          make release-assets
+          make release-assets docker-build-static
       - name: Upload linux_amd64.tar.gz
         if: hashFiles('linux_amd64.tar.gz') != ''
         uses: actions/upload-release-asset@v1
@@ -41,6 +41,14 @@ jobs:
           asset_path: ./linux_amd64_dbg.tar.gz
           asset_name: linux_amd64_dbg.tar.gz
           asset_content_type: application/tar+gzip
+      - name: Upload linux_amd64_static.tar.gz
+        if: hashFiles('linux_amd64_static.tar.gz') != ''
+        uses: actions/upload-release-asset@v1
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: ./linux_amd64_static.tar.gz
+          asset_name: linux_amd64_static.tar.gz
+          asset_content_type: application/tar+gzip
       - name: Upload linux_arm64.tar.gz
         if: hashFiles('linux_arm64.tar.gz') != ''
         uses: actions/upload-release-asset@v1

Makefile

@@ -42,3 +42,8 @@ BUILD_LDFLAGS=-s -w
 
 ## Run tests
 test: test-unit
+
+# Build statically linked binary in docker.
+docker-build-static:
+	docker run --platform linux/amd64 -v $(PWD):/code -w /code --rm golang:alpine /bin/sh -c 'apk add build-base && GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build --ldflags "-linkmode external -extldflags=-static" -tags cgo_zstd -o ./bin/catp ./cmd/catp && ./bin/catp'
+	cd ./bin && tar zcvf ../linux_amd64_static.tar.gz *

cmd/catp/README.md

@@ -11,52 +11,53 @@ go install github.com/bool64/progress/cmd/catp@latest
 or download from [releases](https://github.com/bool64/progress/releases).
 
 ```
-wget https://github.com/bool64/progress/releases/latest/download/linux_amd64.tar.gz && tar xf linux_amd64.tar.gz && rm linux_amd64.tar.gz
+wget https://github.com/bool64/progress/releases/latest/download/linux_amd64_static.tar.gz && tar xf linux_amd64_static.tar.gz && rm linux_amd64_static.tar.gz
 ./catp -version
 ```
 
 ## Usage
 
 ```
-catp dev, go1.22rc1 CGO_ZSTD
-
-catp prints contents of files to STDOUT or dir/file output,
-while printing current progress status to STDERR.
+catp prints contents of files to STDOUT or dir/file output, 
+while printing current progress status to STDERR. 
 It can decompress data from .gz and .zst files.
 Use dash (-) as PATH to read STDIN.
 
 Usage of catp:
 catp [OPTIONS] PATH ...
   -dbg-cpu-prof string
-    	write first 10 seconds of CPU profile to file
+        write first 10 seconds of CPU profile to file
   -dbg-mem-prof string
-    	write heap profile to file after 10 seconds
+        write heap profile to file after 10 seconds
+  -l    count lines
   -no-progress
-    	disable progress printing
+        disable progress printing
   -out-dir string
-    	output to directory instead of STDOUT
-    	files will be written to out dir with original base names
-    	disables output flag
+        output to directory instead of STDOUT
+        files will be written to out dir with original base names
+        disables output flag
   -output string
-    	output to file (can have .gz or .zst ext for compression) instead of STDOUT
+        output to file (can have .gz or .zst ext for compression) instead of STDOUT
   -parallel int
-    	number of parallel readers if multiple files are provided
-    	lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file)
-    	use 0 for multi-threaded zst decoder (slightly faster at cost of more CPU) (default 1)
+        number of parallel readers if multiple files are provided
+        lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file)
+        use 0 for multi-threaded zst decoder (slightly faster at cost of more CPU) (default 1)
   -pass value
-    	filter matching, may contain multiple AND patterns separated by ^,
-    	if filter matches, line is passed to the output (unless filtered out by -skip)
-    	each -pass value is added with OR logic,
-    	for example, you can use "-pass bar^baz -pass foo" to only keep lines that have (bar AND baz) OR foo
+        filter matching, may contain multiple AND patterns separated by ^,
+        if filter matches, line is passed to the output (unless filtered out by -skip)
+        each -pass value is added with OR logic,
+        for example, you can use "-pass bar^baz -pass foo" to only keep lines that have (bar AND baz) OR foo
   -progress-json string
-    	write current progress to a file
+        write current progress to a file
+  -rate-limit float
+        output rate limit lines per second
   -skip value
-    	filter matching, may contain multiple AND patterns separated by ^,
-    	if filter matches, line is removed from the output (even if it passed -pass)
-    	each -skip value is added with OR logic,
-    	for example, you can use "-skip quux^baz -skip fooO" to skip lines that have (quux AND baz) OR fooO
+        filter matching, may contain multiple AND patterns separated by ^,
+        if filter matches, line is removed from the output (even if it passed -pass)
+        each -skip value is added with OR logic,
+        for example, you can use "-skip quux^baz -skip fooO" to skip lines that have (quux AND baz) OR fooO
   -version
-    	print version and exit
+        print version and exit
 ```
 
 ## Examples

cmd/catp/catp/app.go

@@ -4,6 +4,7 @@ package catp
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -23,6 +24,7 @@ import (
 	"github.com/bool64/progress"
 	"github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
+	"golang.org/x/time/rate"
 )
 
 var versionExtra []string
@@ -57,7 +59,11 @@ type runner struct {
 	lastStatusTime        int64
 	lastBytesUncompressed int64
 
+	rateLimit float64
+	limiter   *rate.Limiter
+
 	noProgress bool
+	countLines bool
 
 	hasOptions bool
 	options    Options
@@ -198,10 +204,19 @@ func (r *runner) scanFile(filename string, rd io.Reader, out io.Writer) {
 	lines := 0
 	buf := make([]byte, 64*1024)
 
+	linesPush := 1000
+	if r.rateLimit < 100 {
+		linesPush = 1
+	}
+
 	for s.Scan() {
 		lines++
 
-		if lines >= 1000 {
+		if r.limiter != nil {
+			_ = r.limiter.Wait(context.Background()) //nolint:errcheck // No failure condition here.
+		}
+
+		if lines >= linesPush {
 			atomic.AddInt64(&r.currentLines, int64(lines))
 			lines = 0
 		}
@@ -390,7 +405,11 @@ func (r *runner) cat(filename string) (err error) { //nolint:gocyclo
 		})
 	}
 
-	if len(r.pass) > 0 || len(r.skip) > 0 || r.parallel > 1 || r.hasOptions {
+	if r.rateLimit > 0 {
+		r.limiter = rate.NewLimiter(rate.Limit(r.rateLimit), 100)
+	}
+
+	if len(r.pass) > 0 || len(r.skip) > 0 || r.parallel > 1 || r.hasOptions || r.countLines || r.rateLimit > 0 {
 		r.scanFile(filename, rd, out)
 	} else {
 		r.readFile(rd, out)
@@ -502,6 +521,8 @@ func Main(options ...func(o *Options)) error { //nolint:funlen,cyclop,gocognit,g
 	memProfile := flag.String("dbg-mem-prof", "", "write heap profile to file after 10 seconds")
 	output := flag.String("output", "", "output to file (can have .gz or .zst ext for compression) instead of STDOUT")
 	flag.BoolVar(&r.noProgress, "no-progress", false, "disable progress printing")
+	flag.BoolVar(&r.countLines, "l", false, "count lines")
+	flag.Float64Var(&r.rateLimit, "rate-limit", 0, "output rate limit lines per second")
 	progressJSON := flag.String("progress-json", "", "write current progress to a file")
 	ver := flag.Bool("version", false, "print version and exit")
 

go.mod

@@ -1,10 +1,11 @@
 module github.com/bool64/progress
 
-go 1.22
+go 1.23.0
 
 require (
 	github.com/DataDog/zstd v1.5.7
 	github.com/bool64/dev v0.2.40
 	github.com/klauspost/compress v1.18.0
 	github.com/klauspost/pgzip v1.2.6
+	golang.org/x/time v0.12.0
 )

go.sum

@@ -6,3 +6,5 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=