Update/iterator efficiently: Improving Memory

pkg/adapter/factory.go

@@ -84,7 +84,7 @@ func NewAdapter(ctx *tcontext.TransferMetadata, config types.Config) (map[types.
 		switch types.AdapterType(config.DestinationAdapter) {
 
 		case types.FolderAdapterType:
-			adapters[types.OutputAdapterRole] = &ofolder.FolderAdapter{Role: types.OutputAdapterRole, Uploader: &ofolder.SequentialUploader{}}
+			adapters[types.OutputAdapterRole] = &ofolder.FolderAdapter{Role: types.OutputAdapterRole, Uploader: &ofolder.ParallelUploader{}}
 			outputAdp = "folder"
 
 		case types.InterlynkAdapterType:

pkg/engine/transfer.go

@@ -16,15 +16,11 @@
 package engine
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
-	"strings"
 
 	adapter "github.com/interlynk-io/sbommv/pkg/adapter"
-	"github.com/interlynk-io/sbommv/pkg/converter"
 	"github.com/interlynk-io/sbommv/pkg/iterator"
 	"github.com/interlynk-io/sbommv/pkg/logger"
 	"github.com/interlynk-io/sbommv/pkg/tcontext"
@@ -99,7 +95,7 @@ func dryRun(ctx tcontext.TransferMetadata, sbomIterator iterator.SBOMIterator, i
 	// Step 1: Store SBOMs in memory (avoid consuming iterator)
 	var sboms []*iterator.SBOM
 	for {
-		sbom, err := sbomIterator.Next(ctx.Context)
+		sbom, err := sbomIterator.Next(ctx)
 		if err == io.EOF {
 			break
 		}
@@ -127,100 +123,78 @@ func dryRun(ctx tcontext.TransferMetadata, sbomIterator iterator.SBOMIterator, i
 	return nil
 }
 
-func sbomConversion(sbomIterator iterator.SBOMIterator, transferCtx tcontext.TransferMetadata) []*iterator.SBOM {
-	logger.LogDebug(transferCtx.Context, "Processing SBOM conversion")
-
-	var convertedSBOMs []*iterator.SBOM
-	var totalMinifiedSBOM int
-	var totalSBOM int
-	for {
-		sbom, err := sbomIterator.Next(transferCtx.Context)
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			logger.LogError(transferCtx.Context, err, "Error retrieving SBOM from iterator")
-			continue // Skip erroring SBOMs, proceed with next
-		}
-
-		// Convert SBOM to CycloneDX for Dependency-Track
-		convertedData, err := converter.ConvertSBOM(transferCtx, sbom.Data, converter.FormatCycloneDX)
-		if err != nil {
-			logger.LogInfo(transferCtx.Context, "Failed to convert SBOM to CycloneDX", "file", sbom.Path, "error", err)
-			continue // Skip unconverted SBOMs
-		}
-
-		// let's check minimfied SBOM
-		sbom.Data, totalMinifiedSBOM, err = convertMinifiedJSON(transferCtx, convertedData, totalMinifiedSBOM)
-
-		// Update SBOM data with converted content
-		sbom.Data = convertedData
-
-		if strings.Contains(sbom.Path, "spdx") {
-			sbom.Path = strings.Replace(sbom.Path, "spdx", "spdxtocdx", 1)
-			// transferCtx.FilePath = sbom.Path // Sync FilePath for logging
-		}
-
-		totalSBOM++
-		convertedSBOMs = append(convertedSBOMs, sbom)
-	}
-
-	logger.LogDebug(transferCtx.Context, "Out of total SBOM", "value", totalSBOM, "total minifiedJSONSBOM converted to preety JSON", totalMinifiedSBOM)
-	logger.LogDebug(transferCtx.Context, "Successfully SBOM conversion")
-
-	return convertedSBOMs
-}
+// func sbomConversion(sbomIterator iterator.SBOMIterator, transferCtx tcontext.TransferMetadata) []*iterator.SBOM {
+// 	logger.LogDebug(transferCtx.Context, "Processing SBOM conversion")
+
+// 	var convertedSBOMs []*iterator.SBOM
+// 	var totalMinifiedSBOM int
+// 	var totalSBOM int
+// 	for {
+// 		sbom, err := sbomIterator.Next(transferCtx)
+// 		if err == io.EOF {
+// 			break
+// 		}
+// 		if err != nil {
+// 			logger.LogError(transferCtx.Context, err, "Error retrieving SBOM from iterator")
+// 			continue // Skip erroring SBOMs, proceed with next
+// 		}
+
+// 		// Convert SBOM to CycloneDX for Dependency-Track
+// 		convertedData, err := converter.ConvertSBOM(transferCtx, sbom.Data, converter.FormatCycloneDX)
+// 		if err != nil {
+// 			logger.LogInfo(transferCtx.Context, "Failed to convert SBOM to CycloneDX", "file", sbom.Path, "error", err)
+// 			continue // Skip unconverted SBOMs
+// 		}
+
+// 		// let's check minimfied SBOM
+// 		sbom.Data, totalMinifiedSBOM, err = convertMinifiedJSON(transferCtx, convertedData, totalMinifiedSBOM)
+
+// 		// Update SBOM data with converted content
+// 		sbom.Data = convertedData
+
+// 		if strings.Contains(sbom.Path, "spdx") {
+// 			sbom.Path = strings.Replace(sbom.Path, "spdx", "spdxtocdx", 1)
+// 			// transferCtx.FilePath = sbom.Path // Sync FilePath for logging
+// 		}
+
+// 		totalSBOM++
+// 		convertedSBOMs = append(convertedSBOMs, sbom)
+// 	}
+
+// 	logger.LogDebug(transferCtx.Context, "Out of total SBOM", "value", totalSBOM, "total minifiedJSONSBOM converted to preety JSON", totalMinifiedSBOM)
+// 	logger.LogDebug(transferCtx.Context, "Successfully SBOM conversion")
+
+// 	return convertedSBOMs
+// }
+
+// func sbomProcessing(transferCtx *tcontext.TransferMetadata, config types.Config, sbomIterator iterator.SBOMIterator) iterator.SBOMIterator {
+// 	logger.LogDebug(transferCtx.Context, "Checking adapter eligibility for undergoing conversion layer", "adapter type", config.DestinationAdapter)
+
+// 	// convert sbom to cdx for DTrack adapter only
+// 	if types.AdapterType(config.DestinationAdapter) == types.DtrackAdapterType {
+// 		logger.LogDebug(transferCtx.Context, "Adapter eligible for conversion layer", "adapter type", config.DestinationAdapter)
+
+// 		logger.LogDebug(transferCtx.Context, "SBOM conversion will take place")
+// 		convertedSBOMs := sbomConversion(sbomIterator, *transferCtx)
+
+// 		return iterator.NewMemoryIterator(convertedSBOMs)
+// 	} else {
+// 		logger.LogDebug(transferCtx.Context, "Adapter not eligible for conversion layer", "adapter type", config.DestinationAdapter)
+// 		logger.LogDebug(transferCtx.Context, "SBOM conversion will not take place")
+// 		return sbomIterator
+// 	}
+// }
 
 func sbomProcessing(transferCtx *tcontext.TransferMetadata, config types.Config, sbomIterator iterator.SBOMIterator) iterator.SBOMIterator {
 	logger.LogDebug(transferCtx.Context, "Checking adapter eligibility for undergoing conversion layer", "adapter type", config.DestinationAdapter)
 
-	// convert sbom to cdx for DTrack adapter only
 	if types.AdapterType(config.DestinationAdapter) == types.DtrackAdapterType {
 		logger.LogDebug(transferCtx.Context, "Adapter eligible for conversion layer", "adapter type", config.DestinationAdapter)
-
 		logger.LogDebug(transferCtx.Context, "SBOM conversion will take place")
-		convertedSBOMs := sbomConversion(sbomIterator, *transferCtx)
-
-		return iterator.NewMemoryIterator(convertedSBOMs)
+		return iterator.NewConvertingSBOMIterator(sbomIterator)
 	} else {
 		logger.LogDebug(transferCtx.Context, "Adapter not eligible for conversion layer", "adapter type", config.DestinationAdapter)
 		logger.LogDebug(transferCtx.Context, "SBOM conversion will not take place")
 		return sbomIterator
 	}
 }
-
-func isMinifiedJSON(data []byte) (bool, []byte, []byte, error) {
-	// Try parsing the JSON
-	var jsonData map[string]interface{}
-	err := json.Unmarshal(data, &jsonData)
-	if err != nil {
-		return false, nil, nil, err
-	}
-
-	// Pretty-print the JSON
-	prettyJSON, err := json.MarshalIndent(jsonData, "", "  ")
-	if err != nil {
-		return false, nil, nil, err
-	}
-
-	// Check if original file is minified by comparing bytes
-	if bytes.Equal(data, prettyJSON) {
-		return false, data, prettyJSON, nil // Already formatted
-	}
-
-	return true, data, prettyJSON, nil // Minified JSON detected
-}
-
-func convertMinifiedJSON(transferCtx tcontext.TransferMetadata, data []byte, totalMinifiedSBOM int) ([]byte, int, error) {
-	minified, original, formatted, err := isMinifiedJSON(data)
-	if err != nil {
-		logger.LogError(transferCtx.Context, err, "Error while isMinifiedJSON")
-		return original, totalMinifiedSBOM, nil
-	}
-
-	if minified {
-		totalMinifiedSBOM++
-		return formatted, totalMinifiedSBOM, nil
-	}
-	return original, totalMinifiedSBOM, nil
-}

pkg/iterator/converter.go

@@ -0,0 +1,99 @@
+package iterator
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"strings"
+
+	"github.com/interlynk-io/sbommv/pkg/converter"
+	"github.com/interlynk-io/sbommv/pkg/logger"
+	"github.com/interlynk-io/sbommv/pkg/tcontext"
+)
+
+// ConvertingSBOMIterator wraps an SBOMIterator and converts SBOMs lazily
+type ConvertingSBOMIterator struct {
+	original          SBOMIterator
+	totalMinifiedSBOM int
+	totalSBOM         int
+}
+
+// NewConvertingSBOMIterator creates a new ConvertingSBOMIterator
+func NewConvertingSBOMIterator(original SBOMIterator) SBOMIterator {
+	return &ConvertingSBOMIterator{
+		original: original,
+	}
+}
+
+// Next fetches the next SBOM, converts it, and returns it
+func (it *ConvertingSBOMIterator) Next(transferCtx tcontext.TransferMetadata) (*SBOM, error) {
+	logger.LogDebug(transferCtx.Context, "Next iteration for ConvertingSBOMIterator")
+	sbom, err := it.original.Next(transferCtx)
+	if err != nil {
+		if err == io.EOF {
+			logger.LogDebug(transferCtx.Context, "SBOM conversion summary", "total SBOMs", it.totalSBOM, "total minified JSON SBOMs converted to pretty JSON", it.totalMinifiedSBOM)
+		}
+		return nil, err
+	}
+
+	// Convert SBOM to CycloneDX
+	convertedData, err := converter.ConvertSBOM(transferCtx, sbom.Data, converter.FormatCycloneDX)
+	if err != nil {
+		logger.LogInfo(transferCtx.Context, "Failed to convert SBOM to CycloneDX", "file", sbom.Path, "error", err)
+		return it.Next(transferCtx) // Skip to the next SBOM
+	}
+
+	// Handle minified JSON conversion
+	sbom.Data, it.totalMinifiedSBOM, err = convertMinifiedJSON(transferCtx, convertedData, it.totalMinifiedSBOM)
+	if err != nil {
+		logger.LogInfo(transferCtx.Context, "Failed to handle minified JSON", "file", sbom.Path, "error", err)
+		return it.Next(transferCtx) // Skip to the next SBOM
+	}
+
+	// Update SBOM data with converted content
+	sbom.Data = convertedData
+
+	// Update path if it contains "spdx"
+	if strings.Contains(sbom.Path, "spdx") {
+		sbom.Path = strings.Replace(sbom.Path, "spdx", "spdxtocdx", 1)
+	}
+
+	it.totalSBOM++
+	return sbom, nil
+}
+
+func isMinifiedJSON(data []byte) (bool, []byte, []byte, error) {
+	// Try parsing the JSON
+	var jsonData map[string]interface{}
+	err := json.Unmarshal(data, &jsonData)
+	if err != nil {
+		return false, nil, nil, err
+	}
+
+	// Pretty-print the JSON
+	prettyJSON, err := json.MarshalIndent(jsonData, "", "  ")
+	if err != nil {
+		return false, nil, nil, err
+	}
+
+	// Check if original file is minified by comparing bytes
+	if bytes.Equal(data, prettyJSON) {
+		return false, data, prettyJSON, nil // Already formatted
+	}
+
+	return true, data, prettyJSON, nil // Minified JSON detected
+}
+
+func convertMinifiedJSON(transferCtx tcontext.TransferMetadata, data []byte, totalMinifiedSBOM int) ([]byte, int, error) {
+	minified, original, formatted, err := isMinifiedJSON(data)
+	if err != nil {
+		logger.LogError(transferCtx.Context, err, "Error while isMinifiedJSON")
+		return original, totalMinifiedSBOM, nil
+	}
+
+	if minified {
+		totalMinifiedSBOM++
+		return formatted, totalMinifiedSBOM, nil
+	}
+	return original, totalMinifiedSBOM, nil
+}

pkg/iterator/iterator.go

@@ -16,8 +16,9 @@
 package iterator
 
 import (
-	"context"
 	"io"
+
+	"github.com/interlynk-io/sbommv/pkg/tcontext"
 )
 
 // SBOM represents a single SBOM file
@@ -31,7 +32,7 @@ type SBOM struct {
 
 // SBOMIterator provides a way to lazily fetch SBOMs one by one
 type SBOMIterator interface {
-	Next(ctx context.Context) (*SBOM, error) // Fetch the next SBOM
+	Next(ctx tcontext.TransferMetadata) (*SBOM, error) // Fetch the next SBOM
 }
 
 // MemoryIterator is an iterator that iterates over a preloaded slice of SBOMs.
@@ -49,7 +50,7 @@ func NewMemoryIterator(sboms []*SBOM) SBOMIterator {
 }
 
 // Next retrieves the next SBOM in memory.
-func (it *MemoryIterator) Next(ctx context.Context) (*SBOM, error) {
+func (it *MemoryIterator) Next(ctx tcontext.TransferMetadata) (*SBOM, error) {
 	if it.index >= len(it.sboms) {
 		return nil, io.EOF // No more SBOMs left
 	}

pkg/source/folder/adapter.go

@@ -117,5 +117,5 @@ func (f *FolderAdapter) UploadSBOMs(ctx *tcontext.TransferMetadata, iterator ite
 // DryRun for Folder Adapter: Displays all fetched SBOMs from folder adapter
 func (f *FolderAdapter) DryRun(ctx *tcontext.TransferMetadata, iter iterator.SBOMIterator) error {
 	reporter := NewFolderReporter(false, "")
-	return reporter.DryRun(ctx.Context, iter)
+	return reporter.DryRun(*ctx, iter)
 }

pkg/source/folder/iterator.go

@@ -15,10 +15,10 @@
 package folder
 
 import (
-	"context"
 	"io"
 
 	"github.com/interlynk-io/sbommv/pkg/iterator"
+	"github.com/interlynk-io/sbommv/pkg/tcontext"
 )
 
 // FolderIterator iterates over SBOMs found in a folder
@@ -36,7 +36,7 @@ func NewFolderIterator(sboms []*iterator.SBOM) *FolderIterator {
 }
 
 // Next retrieves the next SBOM in the iteration
-func (it *FolderIterator) Next(ctx context.Context) (*iterator.SBOM, error) {
+func (it *FolderIterator) Next(ctx tcontext.TransferMetadata) (*iterator.SBOM, error) {
 	if it.index >= len(it.sboms) {
 		return nil, io.EOF
 	}