file_indexer.go

Documentation: github.com/redhat-developer/odo/pkg/util

     1  package util
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"os"
     7  	"path/filepath"
     8  	"strings"
     9  	"time"
    10  
    11  	dfutil "github.com/devfile/library/v2/pkg/util"
    12  
    13  	"github.com/redhat-developer/odo/pkg/testingutil/filesystem"
    14  
    15  	gitignore "github.com/sabhiram/go-gitignore"
    16  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    17  	"k8s.io/klog"
    18  )
    19  
    20  const DotOdoDirectory = ".odo"
    21  const fileIndexName = "odo-file-index.json"
    22  const DotGitIgnoreFile = ".gitignore"
    23  
    24  // FileIndex holds the file index used for storing local file state change
    25  type FileIndex struct {
    26  	metav1.TypeMeta
    27  	Files map[string]FileData
    28  }
    29  
    30  // NewFileIndex returns a fileIndex
    31  func NewFileIndex() *FileIndex {
    32  
    33  	return &FileIndex{
    34  		TypeMeta: metav1.TypeMeta{
    35  			Kind:       "FileIndex",
    36  			APIVersion: "v1",
    37  		},
    38  		Files: make(map[string]FileData),
    39  	}
    40  }
    41  
    42  type FileData struct {
    43  	Size             int64
    44  	LastModifiedDate time.Time
    45  	RemoteAttribute  string `json:"RemoteAttribute,omitempty"`
    46  }
    47  
    48  // ReadFileIndex tries to read the odo index file from the given location and returns the data from the file
    49  // if no such file is present, it means the folder hasn't been walked and thus returns an empty list
    50  func ReadFileIndex(filePath string) (*FileIndex, error) {
    51  	// Read operation
    52  	var fi FileIndex
    53  	if _, err := os.Stat(filePath); os.IsNotExist(err) {
    54  		return NewFileIndex(), nil
    55  	}
    56  
    57  	byteValue, err := os.ReadFile(filePath)
    58  	if err != nil {
    59  		return nil, err
    60  	}
    61  	// Unmarshals the byte values and fill up the file read map
    62  	err = json.Unmarshal(byteValue, &fi)
    63  	if err != nil {
    64  		// This is added here for backward compatibility because
    65  		// if the marshalling fails then that means we are trying to
    66  		// read a very old version of the index and hence we can just
    67  		// ignore it and reset index
    68  		// TODO: we need to remove this later
    69  		return NewFileIndex(), nil
    70  	}
    71  	return &fi, nil
    72  }
    73  
    74  // ResolveIndexFilePath resolves the filepath of the odo index file in the .odo folder
    75  func ResolveIndexFilePath(directory string) (string, error) {
    76  	directoryFi, err := os.Stat(filepath.Join(directory))
    77  	if err != nil {
    78  		return "", err
    79  	}
    80  
    81  	switch mode := directoryFi.Mode(); {
    82  	case mode.IsDir():
    83  		// do directory stuff
    84  		return filepath.Join(directory, DotOdoDirectory, fileIndexName), nil
    85  	case mode.IsRegular():
    86  		// do file stuff
    87  		// for binary files
    88  		return filepath.Join(filepath.Dir(directory), DotOdoDirectory, fileIndexName), nil
    89  	}
    90  
    91  	return directory, nil
    92  }
    93  
    94  // GetIndexFileRelativeToContext returns the index file relative to context i.e.; .odo/odo-file-index.json
    95  func GetIndexFileRelativeToContext() string {
    96  	return filepath.Join(DotOdoDirectory, fileIndexName)
    97  }
    98  
    99  // AddOdoFileIndex adds odo-file-index.json to .gitignore
   100  func AddOdoDirectory(gitIgnoreFile string) error {
   101  	return addOdoDirectory(gitIgnoreFile, filesystem.DefaultFs{})
   102  }
   103  
   104  func addOdoDirectory(gitIgnoreFile string, fs filesystem.Filesystem) error {
   105  	return addFileToIgnoreFile(gitIgnoreFile, DotOdoDirectory, fs)
   106  }
   107  
   108  // TouchGitIgnoreFile checks .gitignore file exists or not, if not then creates it.
   109  // The first return value is the path to the .gitignore file, and the second return value indicates whether the file
   110  // has been created (because it did not exist at the time this function was called).
   111  func TouchGitIgnoreFile(directory string) (gitIgnoreFile string, isNewFile bool, err error) {
   112  	return touchGitIgnoreFile(directory, filesystem.DefaultFs{})
   113  }
   114  
   115  func touchGitIgnoreFile(directory string, fs filesystem.Filesystem) (gitIgnoreFile string, isNewFile bool, err error) {
   116  	_, err = fs.Stat(directory)
   117  	if err != nil {
   118  		return "", false, err
   119  	}
   120  
   121  	gitIgnoreFile = filepath.Join(directory, DotGitIgnoreFile)
   122  
   123  	// err checks the existence of .gitignore and then creates it if it does not exist
   124  	if _, err = fs.Stat(gitIgnoreFile); os.IsNotExist(err) {
   125  		var f filesystem.File
   126  		f, err = fs.OpenFile(gitIgnoreFile, os.O_WRONLY|os.O_CREATE, 0600)
   127  		if err != nil {
   128  			return gitIgnoreFile, false, fmt.Errorf("failed to create .gitignore file: %w", err)
   129  		}
   130  		defer f.Close()
   131  		isNewFile = true
   132  	}
   133  
   134  	return gitIgnoreFile, isNewFile, nil
   135  }
   136  
   137  // DeleteIndexFile deletes the index file. It doesn't throw error if it doesn't exist
   138  func DeleteIndexFile(directory string) error {
   139  	indexFile, err := ResolveIndexFilePath(directory)
   140  	if os.IsNotExist(err) {
   141  		return nil
   142  	} else if err != nil {
   143  		return err
   144  	}
   145  	return dfutil.DeletePath(indexFile)
   146  }
   147  
   148  // IndexerRet is a struct that represent return value of RunIndexer function
   149  type IndexerRet struct {
   150  	FilesChanged  []string
   151  	FilesDeleted  []string
   152  	RemoteDeleted []string
   153  	NewFileMap    map[string]FileData
   154  	ResolvedPath  string
   155  }
   156  
   157  // CalculateFileDataKeyFromPath converts an absolute path to relative (and converts to OS-specific paths) for use
   158  // as a map key in IndexerRet and FileIndex
   159  func CalculateFileDataKeyFromPath(absolutePath string, rootDirectory string) (string, error) {
   160  
   161  	rootDirectory = filepath.FromSlash(rootDirectory)
   162  
   163  	relativeFilename, err := filepath.Rel(rootDirectory, absolutePath)
   164  	if err != nil {
   165  		return "", err
   166  	}
   167  
   168  	return relativeFilename, nil
   169  }
   170  
   171  // GenerateNewFileDataEntry creates a new FileData entry for use by IndexerRet and/or FileIndex
   172  func GenerateNewFileDataEntry(absolutePath string, rootDirectory string) (string, *FileData, error) {
   173  
   174  	relativeFilename, err := CalculateFileDataKeyFromPath(absolutePath, rootDirectory)
   175  	if err != nil {
   176  		return "", nil, err
   177  	}
   178  
   179  	fi, err := os.Stat(absolutePath)
   180  
   181  	if err != nil {
   182  		return "", nil, err
   183  	}
   184  	return relativeFilename, &FileData{
   185  		Size:             fi.Size(),
   186  		LastModifiedDate: fi.ModTime(),
   187  	}, nil
   188  }
   189  
   190  // write writes the map of walked files and info about them, in a file
   191  // filePath is the location of the file to which it is supposed to be written
   192  func write(filePath string, fi *FileIndex) error {
   193  	jsonData, err := json.Marshal(fi)
   194  	if err != nil {
   195  		return err
   196  	}
   197  	// While 0666 is the mask used when a file is created using os.Create,
   198  	// gosec objects, so use 0600 instead
   199  	return os.WriteFile(filePath, jsonData, 0600)
   200  }
   201  
   202  // WriteFile writes a file map to a file, the file map is given by
   203  // newFileMap param and the file location is resolvedPath param
   204  func WriteFile(newFileMap map[string]FileData, resolvedPath string) error {
   205  	newfi := NewFileIndex()
   206  	newfi.Files = newFileMap
   207  	err := write(resolvedPath, newfi)
   208  
   209  	return err
   210  }
   211  
   212  // RunIndexerWithRemote reads the existing index from the given directory and runs the indexer on it
   213  // with the given ignore rules
   214  // it also adds the file index to the .gitignore file and resolves the path
   215  func RunIndexerWithRemote(directory string, originalIgnoreRules []string, remoteDirectories map[string]string) (ret IndexerRet, err error) {
   216  	directory = filepath.FromSlash(directory)
   217  	ret.ResolvedPath, err = ResolveIndexFilePath(directory)
   218  	if err != nil {
   219  		return ret, err
   220  	}
   221  
   222  	// read the odo index file
   223  	existingFileIndex, err := ReadFileIndex(ret.ResolvedPath)
   224  	if err != nil {
   225  		return ret, err
   226  	}
   227  
   228  	returnedIndex, err := runIndexerWithExistingFileIndex(directory, originalIgnoreRules, remoteDirectories, existingFileIndex)
   229  	if err != nil {
   230  		return IndexerRet{}, err
   231  	}
   232  	returnedIndex.ResolvedPath = ret.ResolvedPath
   233  	return returnedIndex, nil
   234  }
   235  
   236  // runIndexerWithExistingFileIndex visits the given directory and creates the new index data
   237  // it ignores the files and folders satisfying the ignoreRules
   238  func runIndexerWithExistingFileIndex(directory string, ignoreRules []string, remoteDirectories map[string]string, existingFileIndex *FileIndex) (ret IndexerRet, err error) {
   239  	destPath := ""
   240  	srcPath := directory
   241  
   242  	ret.NewFileMap = make(map[string]FileData)
   243  
   244  	fileChanged := make(map[string]bool)
   245  	filesDeleted := make(map[string]bool)
   246  	fileRemoteChanged := make(map[string]bool)
   247  
   248  	if len(remoteDirectories) == 0 {
   249  		// The file could be a regular file or even a folder, so use recursiveTar which handles symlinks, regular files and folders
   250  		pathOptions := recursiveCheckerPathOptions{directory, filepath.Dir(srcPath), filepath.Base(srcPath), filepath.Dir(destPath), filepath.Base(destPath)}
   251  		innerRet, err := recursiveChecker(pathOptions, ignoreRules, remoteDirectories, *existingFileIndex)
   252  
   253  		if err != nil {
   254  			return IndexerRet{}, err
   255  		}
   256  
   257  		for k, v := range innerRet.NewFileMap {
   258  			ret.NewFileMap[k] = v
   259  		}
   260  
   261  		for _, remote := range innerRet.FilesChanged {
   262  			fileChanged[remote] = true
   263  		}
   264  
   265  		for _, remote := range innerRet.RemoteDeleted {
   266  			fileRemoteChanged[remote] = true
   267  		}
   268  
   269  		for _, remote := range innerRet.FilesDeleted {
   270  			filesDeleted[remote] = true
   271  		}
   272  	}
   273  
   274  	for remoteAttribute := range remoteDirectories {
   275  		path := globEscape(filepath.Join(directory, remoteAttribute))
   276  		matches, err := filepath.Glob(path)
   277  		if err != nil {
   278  			return IndexerRet{}, err
   279  		}
   280  		if len(matches) == 0 {
   281  			return IndexerRet{}, fmt.Errorf("path %q doesn't exist", remoteAttribute)
   282  		}
   283  		for _, fileName := range matches {
   284  			if checkFileExist(fileName) {
   285  				// Fetch path of source file relative to that of source base path so that it can be passed to recursiveTar
   286  				// which uses path relative to base path for taro header to correctly identify file location when untarred
   287  
   288  				// now that the file exists, now we need to get the absolute path
   289  				fileAbsolutePath, err := dfutil.GetAbsPath(fileName)
   290  				if err != nil {
   291  					return IndexerRet{}, err
   292  				}
   293  				klog.V(4).Infof("Got abs path: %s", fileAbsolutePath)
   294  				klog.V(4).Infof("Making %s relative to %s", srcPath, fileAbsolutePath)
   295  
   296  				// We use "FromSlash" to make this OS-based (Windows uses \, Linux & macOS use /)
   297  				// we get the relative path by joining the two
   298  				destFile, err := filepath.Rel(filepath.FromSlash(srcPath), filepath.FromSlash(fileAbsolutePath))
   299  				if err != nil {
   300  					return IndexerRet{}, err
   301  				}
   302  
   303  				// Now we get the source file and join it to the base directory.
   304  				srcFile := filepath.Join(filepath.Base(srcPath), destFile)
   305  
   306  				if value, ok := remoteDirectories[filepath.ToSlash(destFile)]; ok {
   307  					destFile = value
   308  				}
   309  
   310  				klog.V(4).Infof("makeTar srcFile: %s", srcFile)
   311  				klog.V(4).Infof("makeTar destFile: %s", destFile)
   312  
   313  				// The file could be a regular file or even a folder, so use recursiveTar which handles symlinks, regular files and folders
   314  				pathOptions := recursiveCheckerPathOptions{directory, filepath.Dir(srcPath), srcFile, filepath.Dir(destPath), destFile}
   315  				innerRet, err := recursiveChecker(pathOptions, ignoreRules, remoteDirectories, *existingFileIndex)
   316  				if err != nil {
   317  					return IndexerRet{}, err
   318  				}
   319  
   320  				for k, v := range innerRet.NewFileMap {
   321  					ret.NewFileMap[k] = v
   322  				}
   323  
   324  				for _, remote := range innerRet.FilesChanged {
   325  					fileChanged[remote] = true
   326  				}
   327  
   328  				for _, remote := range innerRet.RemoteDeleted {
   329  					fileRemoteChanged[remote] = true
   330  				}
   331  
   332  				for _, remote := range innerRet.FilesDeleted {
   333  					filesDeleted[remote] = true
   334  				}
   335  			} else {
   336  				return IndexerRet{}, fmt.Errorf("path %q doesn't exist", fileName)
   337  			}
   338  		}
   339  	}
   340  
   341  	// find files which are deleted/renamed
   342  	for fileName, value := range existingFileIndex.Files {
   343  		if _, ok := ret.NewFileMap[fileName]; !ok {
   344  			klog.V(4).Infof("Deleting file: %s", fileName)
   345  
   346  			if value.RemoteAttribute != "" {
   347  				currentRemote := value.RemoteAttribute
   348  				for _, remote := range findRemoteFolderForDeletion(currentRemote, remoteDirectories) {
   349  					fileRemoteChanged[remote] = true
   350  				}
   351  			} else {
   352  				ignoreMatcher := gitignore.CompileIgnoreLines(ignoreRules...)
   353  				matched := ignoreMatcher.MatchesPath(fileName)
   354  				if matched {
   355  					continue
   356  				}
   357  				filesDeleted[fileName] = true
   358  			}
   359  		}
   360  	}
   361  
   362  	if len(fileRemoteChanged) > 0 {
   363  		ret.RemoteDeleted = []string{}
   364  	}
   365  	if len(fileChanged) > 0 {
   366  		ret.FilesChanged = []string{}
   367  	}
   368  	if len(filesDeleted) > 0 {
   369  		ret.FilesDeleted = []string{}
   370  	}
   371  	for remote := range fileRemoteChanged {
   372  		ret.RemoteDeleted = append(ret.RemoteDeleted, remote)
   373  	}
   374  	for remote := range fileChanged {
   375  		ret.FilesChanged = append(ret.FilesChanged, remote)
   376  	}
   377  	for remote := range filesDeleted {
   378  		ret.FilesDeleted = append(ret.FilesDeleted, remote)
   379  	}
   380  
   381  	return ret, nil
   382  }
   383  
   384  // recursiveCheckerPathOptions are the path options for the recursiveChecker function
   385  type recursiveCheckerPathOptions struct {
   386  	// directory of the component
   387  	// srcBase is the base of the file/folder
   388  	// srcFile is the file name
   389  	// destBase is the base of the file's/folder's destination
   390  	// destFile is the base of the file's destination
   391  	directory, srcBase, srcFile, destBase, destFile string
   392  }
   393  
   394  // recursiveChecker visits the current source and it's inner files and folders, if any
   395  // the destination values are used to record the appropriate remote location for file or folder
   396  // ignoreRules are used to ignore file and folders
   397  // remoteDirectories are used to find the remote destination of the file/folder and to delete files/folders left behind after the attributes are changed
   398  // existingFileIndex is used to check for file/folder changes
   399  func recursiveChecker(pathOptions recursiveCheckerPathOptions, ignoreRules []string, remoteDirectories map[string]string, existingFileIndex FileIndex) (IndexerRet, error) {
   400  	klog.V(4).Infof("recursiveTar arguments: srcBase: %s, srcFile: %s, destBase: %s, destFile: %s", pathOptions.srcBase, pathOptions.srcFile, pathOptions.destBase, pathOptions.destFile)
   401  
   402  	// The destination is a LINUX container and thus we *must* use ToSlash in order
   403  	// to get the copying over done correctly..
   404  	pathOptions.destBase = filepath.ToSlash(pathOptions.destBase)
   405  	pathOptions.destFile = filepath.ToSlash(pathOptions.destFile)
   406  	klog.V(4).Infof("Corrected destinations: base: %s file: %s", pathOptions.destBase, pathOptions.destFile)
   407  
   408  	joinedPath := filepath.Join(pathOptions.srcBase, pathOptions.srcFile)
   409  	matchedPathsDir, err := filepath.Glob(globEscape(joinedPath))
   410  	if err != nil {
   411  		return IndexerRet{}, err
   412  	}
   413  
   414  	if len(matchedPathsDir) == 0 {
   415  		return IndexerRet{}, fmt.Errorf("path %q doesn't exist", joinedPath)
   416  	}
   417  
   418  	joinedRelPath, err := filepath.Rel(pathOptions.directory, joinedPath)
   419  	if err != nil {
   420  		return IndexerRet{}, err
   421  	}
   422  
   423  	var ret IndexerRet
   424  	ret.NewFileMap = make(map[string]FileData)
   425  
   426  	fileChanged := make(map[string]bool)
   427  	fileRemoteChanged := make(map[string]bool)
   428  
   429  	ignoreMatcher := gitignore.CompileIgnoreLines(ignoreRules...)
   430  
   431  	for _, matchedPath := range matchedPathsDir {
   432  		stat, err := os.Stat(matchedPath)
   433  		if err != nil {
   434  			return IndexerRet{}, err
   435  		}
   436  
   437  		// check if it matches a ignore rule
   438  		rel, err := filepath.Rel(pathOptions.directory, matchedPath)
   439  		if err != nil {
   440  			return IndexerRet{}, err
   441  		}
   442  		match := ignoreMatcher.MatchesPath(rel)
   443  		// the folder matches a glob rule and thus should be skipped
   444  		if match {
   445  			return IndexerRet{}, nil
   446  		}
   447  
   448  		if joinedRelPath != "." {
   449  			// check for changes in the size and the modified date of the file or folder
   450  			// and if the file is newly added
   451  			if _, ok := existingFileIndex.Files[joinedRelPath]; !ok {
   452  				fileChanged[matchedPath] = true
   453  				klog.V(4).Infof("file added: %s", matchedPath)
   454  			} else if !stat.ModTime().Equal(existingFileIndex.Files[joinedRelPath].LastModifiedDate) {
   455  				fileChanged[matchedPath] = true
   456  				klog.V(4).Infof("last modified date changed: %s", matchedPath)
   457  			} else if stat.Size() != existingFileIndex.Files[joinedRelPath].Size {
   458  				fileChanged[matchedPath] = true
   459  				klog.V(4).Infof("size changed: %s", matchedPath)
   460  			}
   461  		}
   462  
   463  		if stat.IsDir() {
   464  
   465  			if stat.Name() == DotOdoDirectory {
   466  				return IndexerRet{}, nil
   467  			}
   468  
   469  			if joinedRelPath != "." {
   470  				folderData, folderChangedData, folderRemoteChangedData := handleRemoteDataFolder(pathOptions.destFile, matchedPath, joinedRelPath, remoteDirectories, existingFileIndex)
   471  				folderData.Size = stat.Size()
   472  				folderData.LastModifiedDate = stat.ModTime()
   473  				ret.NewFileMap[joinedRelPath] = folderData
   474  
   475  				for data, value := range folderChangedData {
   476  					fileChanged[data] = value
   477  				}
   478  
   479  				for data, value := range folderRemoteChangedData {
   480  					fileRemoteChanged[data] = value
   481  				}
   482  			}
   483  
   484  			// read the current folder and read inner files and folders
   485  			entries, err := os.ReadDir(matchedPath)
   486  			if err != nil {
   487  				return IndexerRet{}, err
   488  			}
   489  			if len(entries) == 0 {
   490  				continue
   491  			}
   492  			for _, entry := range entries {
   493  				f, err := entry.Info()
   494  				if err != nil {
   495  					return IndexerRet{}, err
   496  				}
   497  				if _, ok := remoteDirectories[filepath.Join(joinedRelPath, f.Name())]; ok {
   498  					continue
   499  				}
   500  
   501  				opts := recursiveCheckerPathOptions{pathOptions.directory, pathOptions.srcBase, filepath.Join(pathOptions.srcFile, f.Name()), pathOptions.destBase, filepath.Join(pathOptions.destFile, f.Name())}
   502  				innerRet, err := recursiveChecker(opts, ignoreRules, remoteDirectories, existingFileIndex)
   503  				if err != nil {
   504  					return IndexerRet{}, err
   505  				}
   506  
   507  				for k, v := range innerRet.NewFileMap {
   508  					ret.NewFileMap[k] = v
   509  				}
   510  
   511  				for _, remote := range innerRet.FilesChanged {
   512  					fileChanged[remote] = true
   513  				}
   514  				for _, remote := range innerRet.RemoteDeleted {
   515  					fileRemoteChanged[remote] = true
   516  				}
   517  			}
   518  		} else {
   519  			fileData, fileChangedData, fileRemoteChangedData := handleRemoteDataFile(pathOptions.destFile, matchedPath, joinedRelPath, remoteDirectories, existingFileIndex)
   520  			fileData.Size = stat.Size()
   521  			fileData.LastModifiedDate = stat.ModTime()
   522  			ret.NewFileMap[joinedRelPath] = fileData
   523  
   524  			for data, value := range fileChangedData {
   525  				fileChanged[data] = value
   526  			}
   527  
   528  			for data, value := range fileRemoteChangedData {
   529  				fileRemoteChanged[data] = value
   530  			}
   531  		}
   532  	}
   533  
   534  	// remove duplicates in the records
   535  	if len(fileRemoteChanged) > 0 {
   536  		ret.RemoteDeleted = []string{}
   537  	}
   538  	if len(fileChanged) > 0 {
   539  		ret.FilesChanged = []string{}
   540  	}
   541  	for remote := range fileRemoteChanged {
   542  		ret.RemoteDeleted = append(ret.RemoteDeleted, remote)
   543  	}
   544  	for file := range fileChanged {
   545  		ret.FilesChanged = append(ret.FilesChanged, file)
   546  	}
   547  
   548  	return ret, nil
   549  }
   550  
   551  // handleRemoteDataFile handles remote addition, deletion etc for the given file
   552  func handleRemoteDataFile(destFile, path, relPath string, remoteDirectories map[string]string, existingFileIndex FileIndex) (FileData, map[string]bool, map[string]bool) {
   553  	destFile = filepath.ToSlash(destFile)
   554  	fileChanged := make(map[string]bool)
   555  	fileRemoteChanged := make(map[string]bool)
   556  
   557  	remoteDeletionRequired := false
   558  
   559  	remoteAttribute := destFile
   560  	if len(remoteDirectories) == 0 {
   561  		// if no remote attributes specified currently
   562  		remoteAttribute = ""
   563  		if existingFileIndex.Files[relPath].RemoteAttribute != "" && existingFileIndex.Files[relPath].RemoteAttribute != destFile {
   564  			// remote attribute for the file exists in the index
   565  			// but the value doesn't match the current relative path
   566  			// we need to push the current file again and delete the previous location from the container
   567  
   568  			fileChanged[path] = true
   569  			if existingFileIndex.Files[relPath].RemoteAttribute != "" {
   570  				remoteDeletionRequired = true
   571  			}
   572  		}
   573  	} else {
   574  		if value, ok := remoteDirectories[relPath]; ok {
   575  			remoteAttribute = value
   576  		}
   577  
   578  		if existingFileData, ok := existingFileIndex.Files[relPath]; !ok {
   579  			// if the file data doesn't exist in the existing index, we mark the file for pushing
   580  			fileChanged[path] = true
   581  		} else {
   582  			// if the remote attribute is different in the file data from the existing index
   583  			// and the remote attribute is not same as the current relative path
   584  			// we mark the file for pushing and delete the remote paths
   585  			if existingFileData.RemoteAttribute != remoteAttribute && (remoteAttribute != relPath || existingFileData.RemoteAttribute != "") {
   586  				fileChanged[path] = true
   587  				remoteDeletionRequired = true
   588  			}
   589  		}
   590  	}
   591  
   592  	if remoteDeletionRequired {
   593  		// if remote deletion is required but the remote attribute is empty
   594  		// we use the relative path for deletion
   595  		currentRemote := existingFileIndex.Files[relPath].RemoteAttribute
   596  		if currentRemote == "" {
   597  			currentRemote = relPath
   598  		}
   599  		fileRemoteChanged[currentRemote] = true
   600  		for _, remote := range findRemoteFolderForDeletion(currentRemote, remoteDirectories) {
   601  			fileRemoteChanged[remote] = true
   602  		}
   603  	}
   604  
   605  	return FileData{
   606  		RemoteAttribute: filepath.ToSlash(remoteAttribute),
   607  	}, fileChanged, fileRemoteChanged
   608  }
   609  
   610  // handleRemoteDataFolder handles remote addition, deletion etc for the given folder
   611  func handleRemoteDataFolder(destFile, path, relPath string, remoteDirectories map[string]string, existingFileIndex FileIndex) (FileData, map[string]bool, map[string]bool) {
   612  	destFile = filepath.ToSlash(destFile)
   613  	remoteAttribute := destFile
   614  
   615  	fileChanged := make(map[string]bool)
   616  	fileRemoteChanged := make(map[string]bool)
   617  
   618  	remoteChanged := false
   619  
   620  	if len(remoteDirectories) == 0 {
   621  		remoteAttribute = ""
   622  		// remote attribute for the folder exists in the index
   623  		// but the value doesn't match the current relative path
   624  		// we need to push the current folder again and delete the previous location from the container
   625  
   626  		if existingFileIndex.Files[relPath].RemoteAttribute != "" && existingFileIndex.Files[relPath].RemoteAttribute != destFile {
   627  			fileChanged[path] = true
   628  			if existingFileIndex.Files[relPath].RemoteAttribute != "" {
   629  				remoteChanged = true
   630  			}
   631  		}
   632  	} else {
   633  		if value, ok := remoteDirectories[relPath]; ok {
   634  			remoteAttribute = value
   635  		}
   636  
   637  		if existingFileData, ok := existingFileIndex.Files[relPath]; !ok {
   638  			fileChanged[path] = true
   639  		} else {
   640  			// if the remote attribute is different in the file data from the existing index
   641  			// and the remote attribute is not same as the current relative path
   642  			// we mark the file for pushing and delete the remote paths
   643  			if existingFileData.RemoteAttribute != remoteAttribute && (remoteAttribute != relPath || existingFileData.RemoteAttribute != "") {
   644  				fileChanged[path] = true
   645  				remoteChanged = true
   646  			}
   647  		}
   648  	}
   649  
   650  	if remoteChanged {
   651  		// if remote deletion is required but the remote attribute is empty
   652  		// we use the relative path for deletion
   653  		currentRemote := existingFileIndex.Files[relPath].RemoteAttribute
   654  		if currentRemote == "" {
   655  			currentRemote = relPath
   656  		}
   657  		fileRemoteChanged[currentRemote] = true
   658  		for _, remote := range findRemoteFolderForDeletion(currentRemote, remoteDirectories) {
   659  			fileRemoteChanged[remote] = true
   660  		}
   661  	}
   662  
   663  	return FileData{
   664  		RemoteAttribute: filepath.ToSlash(remoteAttribute),
   665  	}, fileChanged, fileRemoteChanged
   666  }
   667  
   668  // checkFileExist check if given file exists or not
   669  func checkFileExist(fileName string) bool {
   670  	_, err := os.Stat(fileName)
   671  	return !os.IsNotExist(err)
   672  }
   673  
   674  // findRemoteFolderForDeletion finds the remote directories which can be deleted by checking the remoteDirectories map
   675  func findRemoteFolderForDeletion(currentRemote string, remoteDirectories map[string]string) []string {
   676  	var remoteDelete []string
   677  	currentRemote = filepath.ToSlash(currentRemote)
   678  	for currentRemote != "" && currentRemote != "." && currentRemote != "/" {
   679  
   680  		found := false
   681  		for _, remote := range remoteDirectories {
   682  			if strings.HasPrefix(remote, currentRemote+"/") || remote == currentRemote {
   683  				found = true
   684  				break
   685  			}
   686  		}
   687  		if !found {
   688  			remoteDelete = append(remoteDelete, currentRemote)
   689  		}
   690  		currentRemote = filepath.ToSlash(filepath.Clean(filepath.Dir(currentRemote)))
   691  	}
   692  	return remoteDelete
   693  }
   694  
   695  // globEscape escapes characters *?[ so that they are
   696  // not included in any match expressions during a Glob.
   697  // Copied from: https://go-review.googlesource.com/c/go/+/18034/2/src/path/filepath/match.go
   698  func globEscape(path string) string {
   699  	var sects []string
   700  	for _, ch := range path {
   701  		strCh := string(ch)
   702  		switch strCh {
   703  		case "*", "?", "[":
   704  			strCh = "[" + strCh + "]"
   705  		}
   706  		sects = append(sects, strCh)
   707  	}
   708  	return strings.Join(sects, "")
   709  }
   710
View as plain text