WRKLDS-1191: oc rsync: Add --last flag

pkg/cli/rsync/copy_rsync.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path/filepath"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -27,6 +28,9 @@ type rsyncStrategy struct {
 	LocalExecutor  executor
 	RemoteExecutor executor
 	podChecker     podChecker
+
+	Last          uint
+	fileDiscovery fileDiscoverer
 }
 
 // DefaultRsyncRemoteShellToUse generates an command to create a remote shell.
@@ -75,15 +79,45 @@ func NewRsyncStrategy(o *RsyncOptions) CopyStrategy {
 		RemoteExecutor: newRemoteExecutor(o),
 		LocalExecutor:  newLocalExecutor(),
 		podChecker:     podAPIChecker{o.Client, o.Namespace, podName, o.ContainerName, o.Quiet, o.ErrOut},
+		Last:           o.Last,
+		fileDiscovery:  o.fileDiscovery,
 	}
 }
 
 func (r *rsyncStrategy) Copy(source, destination *PathSpec, out, errOut io.Writer) error {
 	klog.V(3).Infof("Copying files with rsync")
+
+	// In case --last is specified, discover the right files and pass them to rsync as an explicit list.
+	var (
+		in  io.Reader
+		dst = destination.RsyncPath()
+	)
+	if r.Last > 0 {
+		filenames, err := r.fileDiscovery.DiscoverFiles(source.Path, r.Last)
+		if err != nil {
+			klog.Infof("Warning: failed to apply --last filtering: %v", err)
+		} else {
+			var b bytes.Buffer
+			for _, filename := range filenames {
+				b.WriteString(filename)
+				b.WriteRune('\n')
+			}
+			in = &b
+
+			// Make dst compatible with what rsync does without --last.
+			dst = filepath.Join(dst, filepath.Base(source.Path))
+
+			klog.V(3).Infof("Applied --last=%d to rsync strategy: using %d files", r.Last, len(filenames))
+		}
+	}
+
 	cmd := append([]string{"rsync"}, r.Flags...)
-	cmd = append(cmd, "-e", r.RshCommand, source.RsyncPath(), destination.RsyncPath())
+	if in != nil {
+		cmd = append(cmd, "--files-from", "-")
+	}
+	cmd = append(cmd, "-e", r.RshCommand, source.RsyncPath(), dst)
 	errBuf := &bytes.Buffer{}
-	err := r.LocalExecutor.Execute(cmd, nil, out, errBuf)
+	err := r.LocalExecutor.Execute(cmd, in, out, errBuf)
 	if isExitError(err) {
 		// Check if pod exists
 		if podCheckErr := r.podChecker.CheckPod(); podCheckErr != nil {

pkg/cli/rsync/copy_rsyncd.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"math/rand"
 	"net"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -67,6 +68,9 @@ type rsyncDaemonStrategy struct {
 	PortForwarder  forwarder
 	LocalExecutor  executor
 
+	Last          uint
+	fileDiscovery fileDiscoverer
+
 	daemonPIDFile   string
 	daemonPort      int
 	localPort       int
@@ -220,20 +224,54 @@ func (s *rsyncDaemonStrategy) stopPortForward() {
 
 func (s *rsyncDaemonStrategy) copyUsingDaemon(source, destination *PathSpec, out, errOut io.Writer) error {
 	klog.V(3).Infof("Copying files with rsync daemon")
+
+	// In case --last is specified, discover the right files and pass them to rsync as an explicit list.
+	var (
+		in  io.Reader
+		dst string
+	)
+	if destination.Local() {
+		dst = destination.RsyncPath()
+	} else {
+		dst = destination.Path
+	}
+	if s.Last > 0 {
+		filenames, err := s.fileDiscovery.DiscoverFiles(source.Path, s.Last)
+		if err != nil {
+			klog.Infof("Warning: failed to apply --last filtering: %v", err)
+		} else {
+			var b bytes.Buffer
+			for _, filename := range filenames {
+				b.WriteString(filename)
+				b.WriteRune('\n')
+			}
+			in = &b
+
+			// Make dst compatible with what rsync does without --last.
+			dst = filepath.Join(dst, filepath.Base(source.Path))
+
+			klog.V(3).Infof("Applied --last=%d to rsync-daemon strategy: using %d files", s.Last, len(filenames))
+		}
+	}
+
 	cmd := append([]string{"rsync"}, s.Flags...)
+	if in != nil {
+		cmd = append(cmd, "--files-from", "-")
+	}
+
 	var sourceArg, destinationArg string
 	if source.Local() {
 		sourceArg = source.RsyncPath()
 	} else {
 		sourceArg = localRsyncURL(s.localPort, remoteLabel, source.Path)
 	}
 	if destination.Local() {
-		destinationArg = destination.RsyncPath()
+		destinationArg = dst
 	} else {
-		destinationArg = localRsyncURL(s.localPort, remoteLabel, destination.Path)
+		destinationArg = localRsyncURL(s.localPort, remoteLabel, dst)
 	}
 	cmd = append(cmd, sourceArg, destinationArg)
-	err := s.LocalExecutor.Execute(cmd, nil, out, errOut)
+	err := s.LocalExecutor.Execute(cmd, in, out, errOut)
 	if err != nil {
 		// Determine whether rsync is present in the pod container
 		testRsyncErr := executeWithLogging(s.RemoteExecutor, testRsyncCommand)
@@ -297,6 +335,8 @@ func NewRsyncDaemonStrategy(o *RsyncOptions) CopyStrategy {
 		RemoteExecutor: remoteExec,
 		LocalExecutor:  newLocalExecutor(),
 		PortForwarder:  forwarder,
+		Last:           o.Last,
+		fileDiscovery:  o.fileDiscovery,
 	}
 }
 

pkg/cli/rsync/copy_tar.go

@@ -30,6 +30,7 @@ type tarStrategy struct {
 	RemoteExecutor executor
 	Includes       []string
 	Excludes       []string
+	NoRecursion    bool
 	IgnoredFlags   []string
 	Flags          []string
 }
@@ -44,15 +45,37 @@ func NewTarStrategy(o *RsyncOptions) CopyStrategy {
 
 	remoteExec := newRemoteExecutor(o)
 
+	// Handle --last option by discovering N most recently modified files.
+	includes := append([]string(nil), o.RsyncInclude...)
+	noRecursion := false
+	if o.Last > 0 {
+		if o.Source.Local() {
+			klog.Info("Warning: --last flag is ignored when creating a local tar file")
+		} else {
+			filenames, err := o.fileDiscovery.DiscoverFiles(o.Source.Path, o.Last)
+			if err != nil {
+				klog.Infof("Warning: failed to apply --last filtering: %v", err)
+			} else {
+				// Replace any existing includes with our filtered list.
+				if len(filenames) > 0 {
+					includes = filenames
+					noRecursion = true
+					klog.V(3).Infof("Applied --last=%d to tar strategy: using %d files", o.Last, len(filenames))
+				}
+			}
+		}
+	}
+
 	return &tarStrategy{
 		Quiet:          o.Quiet,
 		Delete:         o.Delete,
-		Includes:       o.RsyncInclude,
+		Includes:       includes,
 		Excludes:       o.RsyncExclude,
 		Tar:            tarHelper,
 		RemoteExecutor: remoteExec,
 		IgnoredFlags:   ignoredFlags,
 		Flags:          tarFlagsFromOptions(o),
+		NoRecursion:    noRecursion,
 	}
 }
 
@@ -145,7 +168,7 @@ func (r *tarStrategy) Copy(source, destination *PathSpec, out, errOut io.Writer)
 	} else {
 		klog.V(4).Infof("Creating local tar file %s from remote path %s", tmp.Name(), source.Path)
 		errBuf := &bytes.Buffer{}
-		err = tarRemote(r.RemoteExecutor, source.Path, r.Includes, r.Excludes, tmp, errBuf)
+		err = tarRemote(r.RemoteExecutor, source.Path, r.Includes, r.Excludes, r.NoRecursion, tmp, errBuf)
 		if err != nil {
 			if checkTar(r.RemoteExecutor) != nil {
 				return strategySetupError("tar not available in container")
@@ -198,7 +221,7 @@ func (r *tarStrategy) String() string {
 	return "tar"
 }
 
-func tarRemote(exec executor, sourceDir string, includes, excludes []string, out, errOut io.Writer) error {
+func tarRemote(exec executor, sourceDir string, includes, excludes []string, noRecursion bool, out, errOut io.Writer) error {
 	klog.V(4).Infof("Tarring %s remotely", sourceDir)
 
 	exclude := []string{}
@@ -220,7 +243,11 @@ func tarRemote(exec executor, sourceDir string, includes, excludes []string, out
 			include = append(include, path.Join(path.Base(sourceDir), pattern))
 		}
 
-		cmd = []string{"tar", "-C", path.Dir(sourceDir), "-c", path.Base(sourceDir)}
+		cmd = []string{"tar", "-C", path.Dir(sourceDir)}
+		if noRecursion {
+			cmd = append(cmd, "--no-recursion")
+		}
+		cmd = append(cmd, "-c", path.Base(sourceDir))
 		cmd = append(cmd, append(include, exclude...)...)
 	}
 	klog.V(4).Infof("Remote tar command: %s", strings.Join(cmd, " "))

pkg/cli/rsync/copy_tar_test.go

@@ -0,0 +1,77 @@
+package rsync
+
+import (
+	"errors"
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+// TestNewTarStrategy_FileDiscovery tests the specific file discovery logic in NewTarStrategy.
+func TestNewTarStrategy_FileDiscovery(t *testing.T) {
+	testCases := []struct {
+		name             string
+		originalIncludes []string
+		discoveredFiles  []string
+		discoveryError   error
+		expectedIncludes []string
+	}{
+		{
+			name:             "discovery finds files - replaces original includes",
+			originalIncludes: []string{"*.log", "*.txt"},
+			discoveredFiles:  []string{"newest.log", "middle.log", "oldest.log"},
+			expectedIncludes: []string{"newest.log", "middle.log", "oldest.log"},
+		},
+		{
+			name:             "discovery finds no files - keeps original includes",
+			originalIncludes: []string{"*.log", "*.txt"},
+			discoveredFiles:  []string{},
+			expectedIncludes: []string{"*.log", "*.txt"},
+		},
+		{
+			name:             "discovery fails - keeps original includes",
+			originalIncludes: []string{"*.log", "*.txt"},
+			discoveryError:   errors.New("command failed"),
+			expectedIncludes: []string{"*.log", "*.txt"},
+		},
+		{
+			name:             "no original includes but discovery finds files",
+			originalIncludes: []string{},
+			discoveredFiles:  []string{"file1.txt", "file2.txt"},
+			expectedIncludes: []string{"file1.txt", "file2.txt"},
+		},
+		{
+			name:             "no original includes and no discovery",
+			originalIncludes: []string{},
+			discoveredFiles:  []string{},
+			expectedIncludes: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Init the strategy.
+			options := &RsyncOptions{
+				RsyncInclude: tc.originalIncludes,
+				Last:         3, // Enable file discovery
+				Source:       &PathSpec{PodName: "test-pod", Path: "/test/path"},
+				Destination:  &PathSpec{Path: "/local/path"},
+				fileDiscovery: &mockFileDiscoverer{
+					files: tc.discoveredFiles,
+					err:   tc.discoveryError,
+				},
+			}
+
+			strategy := NewTarStrategy(options).(*tarStrategy)
+
+			// Verify the result matches expectations.
+			sort.Strings(strategy.Includes)
+			sort.Strings(tc.expectedIncludes)
+			if !cmp.Equal(strategy.Includes, tc.expectedIncludes) {
+				t.Errorf("expected includes mismatch: \n%s\n",
+					cmp.Diff(tc.expectedIncludes, strategy.Includes))
+			}
+		})
+	}
+}

pkg/cli/rsync/discovery.go

@@ -0,0 +1,7 @@
+package rsync
+
+// fileDiscoverer discovers files at the given path,
+// limiting the list to lastN most recently modified files.
+type fileDiscoverer interface {
+	DiscoverFiles(basePath string, lastN uint) ([]string, error)
+}

pkg/cli/rsync/discovery_local.go

@@ -0,0 +1,65 @@
+package rsync
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"time"
+
+	"k8s.io/klog/v2"
+)
+
+// localFileDiscoverer implements fileDiscoverer interface for local directories.
+type localFileDiscoverer struct{}
+
+func newLocalFileDiscoverer() localFileDiscoverer {
+	return localFileDiscoverer{}
+}
+
+func (discoverer localFileDiscoverer) DiscoverFiles(basePath string, last uint) ([]string, error) {
+	klog.V(4).Infof("Discovering files in local directory %s (last = %d)", basePath, last)
+
+	entries, err := os.ReadDir(basePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read directory %s: %w", basePath, err)
+	}
+
+	type fileInfo struct {
+		name    string
+		modTime time.Time
+	}
+
+	files := make([]fileInfo, 0, len(entries))
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue // Skip directories, only process regular files.
+		}
+
+		info, err := entry.Info()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get file info for %s: %w", entry.Name(), err)
+		}
+
+		files = append(files, fileInfo{
+			name:    entry.Name(),
+			modTime: info.ModTime(),
+		})
+	}
+
+	// Sort by modification time (newest first).
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].modTime.After(files[j].modTime)
+	})
+
+	// Limit to the latest N files.
+	if len(files) > int(last) {
+		files = files[:last]
+	}
+
+	// Extract just the file names (relative paths).
+	result := make([]string, len(files))
+	for i, file := range files {
+		result[i] = file.name
+	}
+	return result, nil
+}