sync.go

Documentation: github.com/redhat-developer/odo/pkg/sync

     1  package sync
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"path/filepath"
     8  	"strings"
     9  
    10  	"github.com/devfile/library/v2/pkg/devfile/generator"
    11  	dfutil "github.com/devfile/library/v2/pkg/util"
    12  	gitignore "github.com/sabhiram/go-gitignore"
    13  
    14  	"github.com/redhat-developer/odo/pkg/exec"
    15  	"github.com/redhat-developer/odo/pkg/platform"
    16  	"github.com/redhat-developer/odo/pkg/util"
    17  
    18  	"k8s.io/klog"
    19  )
    20  
    21  // SyncClient is a platform-agnostic implementation for sync
    22  type SyncClient struct {
    23  	platformClient platform.Client
    24  	execClient     exec.Client
    25  }
    26  
    27  var _ Client = (*SyncClient)(nil)
    28  
    29  // NewSyncClient instantiates a new SyncClient
    30  func NewSyncClient(platformClient platform.Client, execClient exec.Client) *SyncClient {
    31  	return &SyncClient{
    32  		platformClient: platformClient,
    33  		execClient:     execClient,
    34  	}
    35  }
    36  
    37  // SyncFiles does a couple of things:
    38  // if files changed/deleted are passed in from watch, it syncs them to the component
    39  // otherwise, it checks which files have changed and syncs the delta
    40  // it returns a boolean execRequired and an error. execRequired tells us if files have
    41  // changed and devfile execution is required
    42  func (a SyncClient) SyncFiles(ctx context.Context, syncParameters SyncParameters) (bool, error) {
    43  
    44  	// Whether to write the indexer content to the index file path (resolvePath)
    45  	forceWrite := false
    46  
    47  	// Ret from Indexer function
    48  	var ret util.IndexerRet
    49  
    50  	var deletedFiles []string
    51  	var changedFiles []string
    52  	isWatch := len(syncParameters.WatchFiles) > 0 || len(syncParameters.WatchDeletedFiles) > 0
    53  
    54  	// When this function is invoked by watch, the logic is:
    55  	// 1) If this is the first time that watch has called Push (in this OS process), then generate the file index
    56  	//    using the file indexer, and use that to sync files (eg don't use changed/deleted files list from watch at
    57  	//    this stage; these will be found by the indexer run).
    58  	//    - In the watch scenario, we need to first run the indexer for two reasons:
    59  	// 	    - In cases where the index doesn't initially exist, we need to create it (so we can ADD to it in
    60  	//        later calls to SyncFiles(...) )
    61  	// 	    - Even if it does initially exist, there is no guarantee that the remote pod is consistent with it; so
    62  	//        on our first invocation we need to compare the index with the remote pod (by regenerating the index
    63  	//        and using the changes files list from that to sync the results.)
    64  	//
    65  	// 2) For every other push/sync call after the first, don't run the file indexer, instead we use
    66  	//    the watch events to determine what changed, and ensure that the index is then updated based
    67  	//    on the watch events (to ensure future calls are correct)
    68  
    69  	// True if the index was updated based on the deleted/changed files values from the watch (and
    70  	// thus the indexer doesn't need to run), false otherwise
    71  	indexRegeneratedByWatch := false
    72  
    73  	// If watch files are specified _and_ this is not the first call (by this process) to SyncFiles by the watch command, then insert the
    74  	// changed files into the existing file index, and delete removed files from the index
    75  	if isWatch && !syncParameters.DevfileScanIndexForWatch {
    76  
    77  		err := updateIndexWithWatchChanges(syncParameters)
    78  
    79  		if err != nil {
    80  			return false, err
    81  		}
    82  
    83  		changedFiles = syncParameters.WatchFiles
    84  		deletedFiles = syncParameters.WatchDeletedFiles
    85  		deletedFiles, err = dfutil.RemoveRelativePathFromFiles(deletedFiles, syncParameters.Path)
    86  		if err != nil {
    87  			return false, fmt.Errorf("unable to remove relative path from list of changed/deleted files: %w", err)
    88  		}
    89  		indexRegeneratedByWatch = true
    90  
    91  	}
    92  
    93  	if !indexRegeneratedByWatch {
    94  		// Calculate the files to sync
    95  		// Tries to sync the deltas unless it is a forced push
    96  		// if it is a forced push (ForcePush) reset the index to do a full sync
    97  
    98  		// Before running the indexer, make sure the .odo folder exists (or else the index file will not get created)
    99  		odoFolder := filepath.Join(syncParameters.Path, ".odo")
   100  		if _, err := os.Stat(odoFolder); os.IsNotExist(err) {
   101  			err = os.Mkdir(odoFolder, 0750)
   102  			if err != nil {
   103  				return false, fmt.Errorf("unable to create directory: %w", err)
   104  			}
   105  		}
   106  
   107  		// If the pod changed, reset the index, which will cause the indexer to walk the directory
   108  		// tree and resync all local files.
   109  		// If it is a new component, reset index to make sure any previously existing file is cleaned up
   110  		if syncParameters.ForcePush {
   111  			err := util.DeleteIndexFile(syncParameters.Path)
   112  			if err != nil {
   113  				return false, fmt.Errorf("unable to reset the index file: %w", err)
   114  			}
   115  		}
   116  
   117  		// Run the indexer and find the modified/added/deleted/renamed files
   118  		var err error
   119  		ret, err = util.RunIndexerWithRemote(syncParameters.Path, syncParameters.IgnoredFiles, syncParameters.Files)
   120  
   121  		if err != nil {
   122  			return false, fmt.Errorf("unable to run indexer: %w", err)
   123  		}
   124  
   125  		if len(ret.FilesChanged) > 0 || len(ret.FilesDeleted) > 0 {
   126  			forceWrite = true
   127  		}
   128  
   129  		// apply the glob rules from the .gitignore/.odoignore file
   130  		// and ignore the files on which the rules apply and filter them out
   131  		filesChangedFiltered, filesDeletedFiltered, err := filterIgnores(syncParameters.Path, ret.FilesChanged, ret.FilesDeleted, syncParameters.IgnoredFiles)
   132  		if err != nil {
   133  			return false, err
   134  		}
   135  
   136  		deletedFiles = append(filesDeletedFiltered, ret.RemoteDeleted...)
   137  		deletedFiles = append(deletedFiles, ret.RemoteDeleted...)
   138  		klog.V(4).Infof("List of files to be deleted: +%v", deletedFiles)
   139  		changedFiles = filesChangedFiltered
   140  		klog.V(4).Infof("List of files changed: +%v", changedFiles)
   141  
   142  		if len(filesChangedFiltered) == 0 && len(filesDeletedFiltered) == 0 && !syncParameters.ForcePush {
   143  			return false, nil
   144  		}
   145  
   146  		if syncParameters.ForcePush {
   147  			deletedFiles = append(deletedFiles, "*")
   148  		}
   149  	}
   150  
   151  	err := a.pushLocal(ctx, syncParameters.Path, changedFiles, deletedFiles, syncParameters.ForcePush, syncParameters.IgnoredFiles, syncParameters.CompInfo, ret)
   152  	if err != nil {
   153  		return false, fmt.Errorf("failed to sync to component with name %s: %w", syncParameters.CompInfo.ComponentName, err)
   154  	}
   155  	if forceWrite {
   156  		err = util.WriteFile(ret.NewFileMap, ret.ResolvedPath)
   157  		if err != nil {
   158  			return false, fmt.Errorf("failed to write file: %w", err)
   159  		}
   160  	}
   161  
   162  	return true, nil
   163  }
   164  
   165  // filterIgnores applies the gitignore rules on the filesChanged and filesDeleted and filters them
   166  // returns the filtered results which match any of the gitignore rules
   167  func filterIgnores(path string, filesChanged, filesDeleted, absIgnoreRules []string) (filesChangedFiltered, filesDeletedFiltered []string, err error) {
   168  	ignoreMatcher := gitignore.CompileIgnoreLines(absIgnoreRules...)
   169  	for _, file := range filesChanged {
   170  		// filesChanged are absoute paths
   171  		rel, err := filepath.Rel(path, file)
   172  		if err != nil {
   173  			return nil, nil, fmt.Errorf("path=%q, file=%q, %w", path, file, err)
   174  		}
   175  		match := ignoreMatcher.MatchesPath(rel)
   176  		if !match {
   177  			filesChangedFiltered = append(filesChangedFiltered, file)
   178  		}
   179  	}
   180  
   181  	for _, file := range filesDeleted {
   182  		// filesDeleted are relative paths
   183  		match := ignoreMatcher.MatchesPath(file)
   184  		if !match {
   185  			filesDeletedFiltered = append(filesDeletedFiltered, file)
   186  		}
   187  	}
   188  	return filesChangedFiltered, filesDeletedFiltered, nil
   189  }
   190  
   191  // pushLocal syncs source code from the user's disk to the component
   192  func (a SyncClient) pushLocal(ctx context.Context, path string, files []string, delFiles []string, isForcePush bool, globExps []string, compInfo ComponentInfo, ret util.IndexerRet) error {
   193  	klog.V(4).Infof("Push: componentName: %s, path: %s, files: %s, delFiles: %s, isForcePush: %+v", compInfo.ComponentName, path, files, delFiles, isForcePush)
   194  
   195  	// Edge case: check to see that the path is NOT empty.
   196  	emptyDir, err := dfutil.IsEmpty(path)
   197  	if err != nil {
   198  		return fmt.Errorf("unable to check directory: %s: %w", path, err)
   199  	} else if emptyDir {
   200  		return fmt.Errorf("directory/file %s is empty", path)
   201  	}
   202  
   203  	// Sync the files to the pod
   204  	syncFolder := compInfo.SyncFolder
   205  
   206  	if syncFolder != generator.DevfileSourceVolumeMount {
   207  		// Need to make sure the folder already exists on the component or else sync will fail
   208  		klog.V(4).Infof("Creating %s on the remote container if it doesn't already exist", syncFolder)
   209  		cmdArr := getCmdToCreateSyncFolder(syncFolder)
   210  
   211  		_, _, err = a.execClient.ExecuteCommand(ctx, cmdArr, compInfo.PodName, compInfo.ContainerName, false, nil, nil)
   212  		if err != nil {
   213  			return err
   214  		}
   215  	}
   216  	// If there were any files deleted locally, delete them remotely too.
   217  	if len(delFiles) > 0 {
   218  		cmdArr := getCmdToDeleteFiles(delFiles, syncFolder)
   219  
   220  		_, _, err = a.execClient.ExecuteCommand(ctx, cmdArr, compInfo.PodName, compInfo.ContainerName, false, nil, nil)
   221  		if err != nil {
   222  			return err
   223  		}
   224  	}
   225  
   226  	if !isForcePush {
   227  		if len(files) == 0 && len(delFiles) == 0 {
   228  			return nil
   229  		}
   230  	}
   231  
   232  	if isForcePush || len(files) > 0 {
   233  		klog.V(4).Infof("Copying files %s to pod", strings.Join(files, " "))
   234  		err = a.CopyFile(ctx, path, compInfo, syncFolder, files, globExps, ret)
   235  		if err != nil {
   236  			return fmt.Errorf("unable push files to pod: %w", err)
   237  		}
   238  	}
   239  
   240  	return nil
   241  }
   242  
   243  // updateIndexWithWatchChanges uses the pushParameters.WatchDeletedFiles and pushParamters.WatchFiles to update
   244  // the existing index file; the index file is required to exist when this function is called.
   245  func updateIndexWithWatchChanges(syncParameters SyncParameters) error {
   246  	indexFilePath, err := util.ResolveIndexFilePath(syncParameters.Path)
   247  
   248  	if err != nil {
   249  		return fmt.Errorf("unable to resolve path: %s: %w", syncParameters.Path, err)
   250  	}
   251  
   252  	// Check that the path exists
   253  	_, err = os.Stat(indexFilePath)
   254  	if err != nil {
   255  		// This shouldn't happen: in the watch case, SyncFiles should first be called with 'DevfileScanIndexForWatch' set to true, which
   256  		// will generate the index. Then, all subsequent invocations of SyncFiles will run with 'DevfileScanIndexForWatch' set to false,
   257  		// which will not regenerate the index (merely updating it based on changed files.)
   258  		//
   259  		// If you see this error it means somehow watch's SyncFiles was called without the index being first generated (likely because the
   260  		// above mentioned pushParam wasn't set). See SyncFiles(...) for details.
   261  		return fmt.Errorf("resolved path doesn't exist: %s: %w", indexFilePath, err)
   262  	}
   263  
   264  	// Parse the existing index
   265  	fileIndex, err := util.ReadFileIndex(indexFilePath)
   266  	if err != nil {
   267  		return fmt.Errorf("unable to read index from path: %s: %w", indexFilePath, err)
   268  	}
   269  
   270  	rootDir := syncParameters.Path
   271  
   272  	// Remove deleted files from the existing index
   273  	for _, deletedFile := range syncParameters.WatchDeletedFiles {
   274  
   275  		relativePath, err := util.CalculateFileDataKeyFromPath(deletedFile, rootDir)
   276  
   277  		if err != nil {
   278  			klog.V(4).Infof("Error occurred for %s: %v", deletedFile, err)
   279  			continue
   280  		}
   281  		delete(fileIndex.Files, relativePath)
   282  		klog.V(4).Infof("Removing watch deleted file from index: %s", relativePath)
   283  	}
   284  
   285  	// Add changed files to the existing index
   286  	for _, addedOrModifiedFile := range syncParameters.WatchFiles {
   287  		relativePath, fileData, err := util.GenerateNewFileDataEntry(addedOrModifiedFile, rootDir)
   288  
   289  		if err != nil {
   290  			klog.V(4).Infof("Error occurred for %s: %v", addedOrModifiedFile, err)
   291  			continue
   292  		}
   293  		fileIndex.Files[relativePath] = *fileData
   294  		klog.V(4).Infof("Added/updated watched file in index: %s", relativePath)
   295  	}
   296  
   297  	// Write the result
   298  	return util.WriteFile(fileIndex.Files, indexFilePath)
   299  
   300  }
   301  
   302  // getCmdToCreateSyncFolder returns the command used to create the remote sync folder on the running container
   303  func getCmdToCreateSyncFolder(syncFolder string) []string {
   304  	return []string{"mkdir", "-p", syncFolder}
   305  }
   306  
   307  // getCmdToDeleteFiles returns the command used to delete the remote files on the container that are marked for deletion
   308  func getCmdToDeleteFiles(delFiles []string, syncFolder string) []string {
   309  	rmPaths := dfutil.GetRemoteFilesMarkedForDeletion(delFiles, syncFolder)
   310  	klog.V(4).Infof("remote files marked for deletion are %+v", rmPaths)
   311  	cmdArr := []string{"rm", "-rf"}
   312  
   313  	for _, remote := range rmPaths {
   314  		cmdArr = append(cmdArr, filepath.ToSlash(remote))
   315  	}
   316  	return cmdArr
   317  }
   318
View as plain text