registry.go

Documentation: github.com/redhat-developer/odo/pkg/registry

     1  package registry
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io/fs"
     7  	"path"
     8  	"path/filepath"
     9  	"sort"
    10  	"strings"
    11  	"sync"
    12  
    13  	"github.com/blang/semver"
    14  	devfilev1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
    15  	apidevfile "github.com/devfile/api/v2/pkg/devfile"
    16  	dfutil "github.com/devfile/library/v2/pkg/util"
    17  	indexSchema "github.com/devfile/registry-support/index/generator/schema"
    18  	"github.com/devfile/registry-support/registry-library/library"
    19  	"k8s.io/klog"
    20  
    21  	"github.com/redhat-developer/odo/pkg/api"
    22  	"github.com/redhat-developer/odo/pkg/devfile"
    23  	"github.com/redhat-developer/odo/pkg/devfile/location"
    24  	"github.com/redhat-developer/odo/pkg/kclient"
    25  	"github.com/redhat-developer/odo/pkg/log"
    26  	"github.com/redhat-developer/odo/pkg/preference"
    27  	"github.com/redhat-developer/odo/pkg/segment"
    28  	"github.com/redhat-developer/odo/pkg/testingutil/filesystem"
    29  	"github.com/redhat-developer/odo/pkg/util"
    30  )
    31  
    32  type RegistryClient struct {
    33  	fsys             filesystem.Filesystem
    34  	preferenceClient preference.Client
    35  	kubeClient       kclient.ClientInterface
    36  }
    37  
    38  var _ Client = (*RegistryClient)(nil)
    39  
    40  const (
    41  	CONFLICT_DIR_NAME = "CONFLICT_STARTER_PROJECT"
    42  )
    43  
    44  func NewRegistryClient(fsys filesystem.Filesystem, preferenceClient preference.Client, kubeClient kclient.ClientInterface) RegistryClient {
    45  	return RegistryClient{
    46  		fsys:             fsys,
    47  		preferenceClient: preferenceClient,
    48  		kubeClient:       kubeClient,
    49  	}
    50  }
    51  
    52  // PullStackFromRegistry pulls stack from registry with all stack resources (all media types) to the destination directory
    53  func (o RegistryClient) PullStackFromRegistry(registry string, stack string, destDir string, options library.RegistryOptions) error {
    54  	klog.V(3).Infof("sending telemetry data: %#v", options.Telemetry)
    55  	return library.PullStackFromRegistry(registry, stack, destDir, options)
    56  }
    57  
    58  // DownloadFileInMemory uses the url to download the file and return bytes
    59  func (o RegistryClient) DownloadFileInMemory(params dfutil.HTTPRequestParams) ([]byte, error) {
    60  	return util.DownloadFileInMemory(params)
    61  }
    62  
    63  // DownloadStarterProject downloads a starter project referenced in devfile
    64  // There are 3 cases to consider here:
    65  // Case 1: If there is devfile in the starterproject, replace all the contents of contextDir with that of the starterproject; warn about this
    66  // Case 2: If there is no devfile, and there is no conflict between the contents of contextDir and starterproject, then copy the contents of the starterproject into contextDir.
    67  // Case 3: If there is no devfile, and there is conflict between the contents of contextDir and starterproject, copy contents of starterproject into a dir named CONFLICT_STARTER_PROJECT; warn about this
    68  func (o RegistryClient) DownloadStarterProject(starterProject *devfilev1.StarterProject, decryptedToken string, contextDir string, verbose bool) (containsDevfile bool, err error) {
    69  	// Let the project be downloaded in a temp directory
    70  	starterProjectTmpDir, err := o.fsys.TempDir("", "odostarterproject")
    71  	if err != nil {
    72  		return containsDevfile, err
    73  	}
    74  	defer func() {
    75  		err = o.fsys.RemoveAll(starterProjectTmpDir)
    76  		if err != nil {
    77  			klog.V(2).Infof("failed to delete temporary starter project dir %s; cause: %s", starterProjectTmpDir, err.Error())
    78  		}
    79  	}()
    80  	err = DownloadStarterProject(o.fsys, starterProject, decryptedToken, starterProjectTmpDir, verbose)
    81  	if err != nil {
    82  		return containsDevfile, err
    83  	}
    84  
    85  	// Case 1: If there is devfile in the starterproject, replace all the contents of contextDir with that of the starterproject; warn about this
    86  	if containsDevfile, err = location.DirectoryContainsDevfile(o.fsys, starterProjectTmpDir); err != nil {
    87  		return containsDevfile, err
    88  	}
    89  	if containsDevfile {
    90  		fmt.Println()
    91  		log.Warning("A Devfile is present inside the starter project; replacing the entire content of the current directory with the starter project")
    92  		err = removeDirectoryContents(contextDir, o.fsys)
    93  		if err != nil {
    94  			return containsDevfile, fmt.Errorf("failed to delete contents of the current directory; cause %w", err)
    95  		}
    96  		return containsDevfile, util.CopyDirWithFS(starterProjectTmpDir, contextDir, o.fsys)
    97  	}
    98  
    99  	// Case 2: If there is no devfile, and there is no conflict between the contents of contextDir and starterproject, then copy the contents of the starterproject into contextDir.
   100  	// Case 3: If there is no devfile, and there is conflict between the contents of contextDir and starterproject, copy contents of starterproject into a dir named CONFLICT_STARTER_PROJECT; warn about this
   101  	var conflictingFiles []string
   102  	conflictingFiles, err = getConflictingFiles(starterProjectTmpDir, contextDir, o.fsys)
   103  	if err != nil {
   104  		return containsDevfile, err
   105  	}
   106  
   107  	// Case 2
   108  	if len(conflictingFiles) == 0 {
   109  		return containsDevfile, util.CopyDirWithFS(starterProjectTmpDir, contextDir, o.fsys)
   110  	}
   111  
   112  	// Case 3
   113  	conflictingDirPath := filepath.Join(contextDir, CONFLICT_DIR_NAME)
   114  	err = o.fsys.MkdirAll(conflictingDirPath, 0750)
   115  	if err != nil {
   116  		return containsDevfile, err
   117  	}
   118  
   119  	err = util.CopyDirWithFS(starterProjectTmpDir, conflictingDirPath, o.fsys)
   120  	if err != nil {
   121  		return containsDevfile, err
   122  	}
   123  	fmt.Println()
   124  	log.Warningf("There are conflicting files (%s) between starter project and the current directory, hence the starter project has been copied to %s", strings.Join(conflictingFiles, ", "), conflictingDirPath)
   125  
   126  	return containsDevfile, nil
   127  }
   128  
   129  // removeDirectoryContents attempts to remove dir contents without deleting the directory itself, unlike os.RemoveAll() method
   130  func removeDirectoryContents(path string, fsys filesystem.Filesystem) error {
   131  	dir, err := fsys.ReadDir(path)
   132  	if err != nil {
   133  		return err
   134  	}
   135  	for _, f := range dir {
   136  		// a bit of cheating by using absolute file name to make sure this works with a fake filesystem, especially a memory map which is used by our unit tests
   137  		// memorymap's Name() method trims the full path and returns just the file name, which then becomes impossible to find by the RemoveAll method that looks for prefix
   138  		// See: https://github.com/redhat-developer/odo/blob/d717421494f746a5cb12da135f561d12750935f3/vendor/github.com/spf13/afero/memmap.go#L282
   139  		absFileName := filepath.Join(path, f.Name())
   140  		err = fsys.RemoveAll(absFileName)
   141  		if err != nil {
   142  			return fmt.Errorf("failed to remove %s; cause: %w", absFileName, err)
   143  		}
   144  	}
   145  
   146  	return nil
   147  }
   148  
   149  // getConflictingFiles fetches the contents of the two directories in question and compares them to check for conflicting files.
   150  // it returns a list of conflicting files (if any) along with an error (if any).
   151  func getConflictingFiles(spDir, contextDir string, fsys filesystem.Filesystem) (conflictingFiles []string, err error) {
   152  	var (
   153  		contextDirMap = map[string]struct{}{}
   154  	)
   155  	// walk through the contextDir, trim the file path from the file name and append it to a map
   156  	err = fsys.Walk(contextDir, func(path string, info fs.FileInfo, err error) error {
   157  		if err != nil {
   158  			return fmt.Errorf("failed to fetch contents of dir %s; cause: %w", contextDirMap, err)
   159  		}
   160  		if info.IsDir() {
   161  			return nil
   162  		}
   163  		path = strings.TrimPrefix(path, contextDir)
   164  		contextDirMap[path] = struct{}{}
   165  
   166  		return nil
   167  	})
   168  	if err != nil {
   169  		return nil, fmt.Errorf("failed to walk %s dir; cause: %w", contextDir, err)
   170  	}
   171  
   172  	// walk through the starterproject dir, trim the file path from file name, and check if it exists in the contextDir map;
   173  	// if it does, it is a conflicting file, hence append it to the conflictingFiles list.
   174  	err = fsys.Walk(spDir, func(path string, info fs.FileInfo, err error) error {
   175  		if err != nil {
   176  			return fmt.Errorf("failed to fetch contents of dir %s; cause: %w", spDir, err)
   177  		}
   178  		if info.IsDir() {
   179  			return nil
   180  		}
   181  		path = strings.TrimPrefix(path, spDir)
   182  		if _, ok := contextDirMap[path]; ok {
   183  			conflictingFiles = append(conflictingFiles, path)
   184  		}
   185  		return nil
   186  	})
   187  	if err != nil {
   188  		return nil, fmt.Errorf("failed to walk %s dir; cause: %w", spDir, err)
   189  	}
   190  
   191  	return conflictingFiles, nil
   192  }
   193  
   194  // GetDevfileRegistries gets devfile registries from preference file,
   195  // if registry name is specified return the specific registry, otherwise return all registries
   196  func (o RegistryClient) GetDevfileRegistries(registryName string) ([]api.Registry, error) {
   197  	var allRegistries []api.Registry
   198  
   199  	if o.kubeClient != nil {
   200  		clusterRegistries, err := o.kubeClient.GetRegistryList()
   201  		if err != nil {
   202  			// #6636 : errors should not be blocking
   203  			klog.V(3).Infof("failed to get Devfile registries from the cluster: %v", err)
   204  		} else {
   205  			allRegistries = append(allRegistries, clusterRegistries...)
   206  		}
   207  	}
   208  	allRegistries = append(allRegistries, o.preferenceClient.RegistryList()...)
   209  
   210  	hasName := registryName != ""
   211  	var result []api.Registry
   212  	for _, registry := range allRegistries {
   213  		if hasName {
   214  			if registryName == registry.Name {
   215  				reg := api.Registry{
   216  					Name:   registry.Name,
   217  					URL:    registry.URL,
   218  					Secure: registry.Secure,
   219  				}
   220  				result = append(result, reg)
   221  				return result, nil
   222  			}
   223  			continue
   224  		}
   225  		reg := api.Registry{
   226  			Name:   registry.Name,
   227  			URL:    registry.URL,
   228  			Secure: registry.Secure,
   229  		}
   230  		result = append(result, reg)
   231  	}
   232  
   233  	return result, nil
   234  }
   235  
   236  // ListDevfileStacks lists all the available devfile stacks in devfile registry
   237  // When `withDevfileContent` and `detailsFlag` are both true, another HTTP call is executed to download the Devfile
   238  func (o RegistryClient) ListDevfileStacks(ctx context.Context, registryName, devfileFlag, filterFlag string, detailsFlag bool, withDevfileContent bool) (DevfileStackList, error) {
   239  	catalogDevfileList := &DevfileStackList{}
   240  	var err error
   241  
   242  	// TODO: consider caching registry information for better performance since it should be fairly stable over time
   243  	// Get devfile registries
   244  	catalogDevfileList.DevfileRegistries, err = o.GetDevfileRegistries(registryName)
   245  	if err != nil {
   246  		return *catalogDevfileList, err
   247  	}
   248  	if catalogDevfileList.DevfileRegistries == nil {
   249  		return *catalogDevfileList, nil
   250  	}
   251  
   252  	// first retrieve the indices for each registry, concurrently
   253  	devfileIndicesMutex := &sync.Mutex{}
   254  	retrieveRegistryIndices := util.NewConcurrentTasks(len(catalogDevfileList.DevfileRegistries))
   255  
   256  	// The 2D slice index is the priority of the registry (highest priority has highest index)
   257  	// and the element is the devfile slice that belongs to the registry
   258  	registrySlice := make([][]api.DevfileStack, len(catalogDevfileList.DevfileRegistries))
   259  	for regPriority, reg := range catalogDevfileList.DevfileRegistries {
   260  		// Load the devfile registry index.json
   261  		registry := reg                 // Needed to prevent the lambda from capturing the value
   262  		registryPriority := regPriority // Needed to prevent the lambda from capturing the value
   263  		retrieveRegistryIndices.Add(util.ConcurrentTask{ToRun: func(errChannel chan error) {
   264  			registryDevfiles, err := getRegistryStacks(ctx, registry)
   265  			if err != nil {
   266  				log.Warningf("Registry %s is not set up properly with error: %v, please check the registry URL, and credential and remove add the registry again (refer to `odo preference add registry --help`)\n", registry.Name, err)
   267  				return
   268  			}
   269  
   270  			devfileIndicesMutex.Lock()
   271  			registrySlice[registryPriority] = registryDevfiles
   272  			devfileIndicesMutex.Unlock()
   273  		}})
   274  	}
   275  	if err := retrieveRegistryIndices.Run(); err != nil {
   276  		return *catalogDevfileList, err
   277  	}
   278  
   279  	// Go through all the devfiles and filter based on:
   280  	// What's in the name or description
   281  	// The exact name of the devfile
   282  	for priorityNumber, registryDevfiles := range registrySlice {
   283  
   284  		devfiles := []api.DevfileStack{}
   285  
   286  	devfileLoop:
   287  		for _, devfile := range registryDevfiles {
   288  
   289  			// Add the "priority" of the registry to the devfile
   290  			devfile.Registry.Priority = priorityNumber
   291  
   292  			if filterFlag != "" {
   293  				filters := strings.Split(filterFlag, ",")
   294  				for _, filter := range filters {
   295  					filter = strings.TrimSpace(filter)
   296  					archs := append(make([]string, 0, len(devfile.Architectures)), devfile.Architectures...)
   297  					if len(archs) == 0 {
   298  						// Devfiles with no architectures are compatible with all architectures.
   299  						archs = append(archs,
   300  							string(apidevfile.AMD64),
   301  							string(apidevfile.ARM64),
   302  							string(apidevfile.PPC64LE),
   303  							string(apidevfile.S390X),
   304  						)
   305  					}
   306  					containsArch := func(s string) bool {
   307  						for _, arch := range archs {
   308  							if strings.Contains(arch, s) {
   309  								return true
   310  							}
   311  						}
   312  						return false
   313  					}
   314  					if !strings.Contains(devfile.Name, filter) && !strings.Contains(devfile.Description, filter) && !containsArch(filter) {
   315  						continue devfileLoop
   316  					}
   317  				}
   318  			}
   319  
   320  			if devfileFlag != "" {
   321  				if devfileFlag != devfile.Name {
   322  					continue
   323  				}
   324  			}
   325  
   326  			// We are fetching the Devfile content only when `--details` and `-o json` flags are used
   327  			if detailsFlag && withDevfileContent {
   328  				devfileData, err := o.retrieveDevfileDataFromRegistry(ctx, devfile.Registry.Name, devfile.Name)
   329  				if err != nil {
   330  					return *catalogDevfileList, err
   331  				}
   332  				devfile.DevfileData = &devfileData
   333  			}
   334  
   335  			devfiles = append(devfiles, devfile)
   336  		}
   337  
   338  		catalogDevfileList.Items = append(catalogDevfileList.Items, devfiles...)
   339  	}
   340  
   341  	// Sort catalogDevfileList.Items by:
   342  	// 1. Priority of the registry (highest priority has highest index)
   343  	// 2. Name of the devfile
   344  	sort.Slice(catalogDevfileList.Items[:], func(i, j int) bool {
   345  		if catalogDevfileList.Items[i].Name == catalogDevfileList.Items[j].Name {
   346  			return catalogDevfileList.Items[i].Registry.Priority < catalogDevfileList.Items[j].Registry.Priority
   347  		}
   348  		return catalogDevfileList.Items[i].Name < catalogDevfileList.Items[j].Name
   349  	})
   350  
   351  	return *catalogDevfileList, nil
   352  }
   353  
   354  // getRegistryStacks retrieves the registry's index devfile stack entries
   355  func getRegistryStacks(ctx context.Context, registry api.Registry) ([]api.DevfileStack, error) {
   356  	isGithubregistry, err := IsGithubBasedRegistry(registry.URL)
   357  	if err != nil {
   358  		return nil, err
   359  	}
   360  	if isGithubregistry {
   361  		return nil, &ErrGithubRegistryNotSupported{}
   362  	}
   363  	// OCI-based registry
   364  	options := segment.GetRegistryOptions(ctx)
   365  	options.NewIndexSchema = true
   366  	devfileIndex, err := library.GetRegistryIndex(registry.URL, options, indexSchema.StackDevfileType)
   367  	if err != nil {
   368  		// Fallback to the "old" index
   369  		klog.V(3).Infof("error while accessing the v2index endpoint for registry %s (%s) => falling back to the old index endpoint: %v",
   370  			registry.Name, registry.URL, err)
   371  		options.NewIndexSchema = false
   372  		devfileIndex, err = library.GetRegistryIndex(registry.URL, options, indexSchema.StackDevfileType)
   373  		if err != nil {
   374  			return nil, err
   375  		}
   376  	}
   377  	return createRegistryDevfiles(registry, devfileIndex)
   378  }
   379  
   380  func createRegistryDevfiles(registry api.Registry, devfileIndex []indexSchema.Schema) ([]api.DevfileStack, error) {
   381  	registryDevfiles := make([]api.DevfileStack, 0, len(devfileIndex))
   382  	for _, devfileIndexEntry := range devfileIndex {
   383  		stackDevfile := api.DevfileStack{
   384  			Name:                   devfileIndexEntry.Name,
   385  			DisplayName:            devfileIndexEntry.DisplayName,
   386  			Description:            devfileIndexEntry.Description,
   387  			Registry:               registry,
   388  			Language:               devfileIndexEntry.Language,
   389  			Tags:                   devfileIndexEntry.Tags,
   390  			ProjectType:            devfileIndexEntry.ProjectType,
   391  			DefaultStarterProjects: devfileIndexEntry.StarterProjects,
   392  			DefaultVersion:         devfileIndexEntry.Version,
   393  			Architectures:          devfileIndexEntry.Architectures,
   394  		}
   395  		for _, v := range devfileIndexEntry.Versions {
   396  			if v.Default {
   397  				// There should be only 1 default version. But if there is more than one, the last one will be used.
   398  				stackDevfile.DefaultVersion = v.Version
   399  				stackDevfile.DefaultStarterProjects = v.StarterProjects
   400  			}
   401  			stackDevfile.Versions = append(stackDevfile.Versions, api.DevfileStackVersion{
   402  				IsDefault:       v.Default,
   403  				Version:         v.Version,
   404  				SchemaVersion:   v.SchemaVersion,
   405  				StarterProjects: v.StarterProjects,
   406  				CommandGroups:   v.CommandGroups,
   407  			})
   408  		}
   409  		sort.Slice(stackDevfile.Versions, func(i, j int) bool {
   410  			vi, err := semver.Make(stackDevfile.Versions[i].Version)
   411  			if err != nil {
   412  				return false
   413  			}
   414  			vj, err := semver.Make(stackDevfile.Versions[j].Version)
   415  			if err != nil {
   416  				return false
   417  			}
   418  			return vi.LT(vj)
   419  		})
   420  
   421  		registryDevfiles = append(registryDevfiles, stackDevfile)
   422  	}
   423  
   424  	return registryDevfiles, nil
   425  }
   426  
   427  func (o RegistryClient) retrieveDevfileDataFromRegistry(ctx context.Context, registryName string, devfileName string) (api.DevfileData, error) {
   428  
   429  	// Create random temporary file
   430  	tmpFile, err := o.fsys.TempDir("", "odo")
   431  	if err != nil {
   432  		return api.DevfileData{}, err
   433  	}
   434  	defer func() {
   435  		err = o.fsys.RemoveAll(tmpFile)
   436  		if err != nil {
   437  			klog.V(2).Infof("failed to delete temporary starter project dir %s; cause: %s", tmpFile, err.Error())
   438  		}
   439  	}()
   440  
   441  	registries, err := o.GetDevfileRegistries(registryName)
   442  	if err != nil {
   443  		return api.DevfileData{}, err
   444  	}
   445  	registryOptions := segment.GetRegistryOptions(ctx)
   446  	registryOptions.NewIndexSchema = true
   447  	// Get the file and save it to the temporary file
   448  	// Why do we do that?
   449  	// 1. We need to get the file from the registry
   450  	// 2. The devfile api library does not support saving in memory
   451  	// 3. We need to get the file from the registry and save it to the temporary file
   452  	// 4. We need to read the file from the temporary file, unmarshal it and then return the devfile data
   453  	for _, reg := range registries {
   454  		if reg.Name == registryName {
   455  			err = o.PullStackFromRegistry(reg.URL, devfileName, tmpFile, registryOptions)
   456  			if err != nil {
   457  				return api.DevfileData{}, err
   458  			}
   459  		}
   460  	}
   461  
   462  	// Get the devfile yaml file from the directory
   463  	devfileYamlFile := location.DevfileFilenamesProvider(o.fsys, tmpFile)
   464  
   465  	// Parse and validate the file and return the devfile data
   466  	devfileObj, err := devfile.ParseAndValidateFromFile(path.Join(tmpFile, devfileYamlFile), "", true)
   467  	if err != nil {
   468  		return api.DevfileData{}, err
   469  	}
   470  
   471  	// Convert DevfileObj to DevfileData
   472  	// use api.GetDevfileData to get supported features
   473  	devfileData, err := api.GetDevfileData(devfileObj)
   474  	if err != nil {
   475  		return api.DevfileData{}, err
   476  	}
   477  
   478  	return *devfileData, nil
   479  }
   480
View as plain text