[WIP] Add Spot/On-demand price to table-wide output irrespective of the price filters #78
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,7 @@ | ||
| ### Go ### | ||
| # IDE | ||
| .idea | ||
|
|
||
| # Binaries for programs and plugins | ||
| *.exe | ||
| *.exe~ | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,14 +2,17 @@ package ec2pricing | |
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/aws/aws-sdk-go/aws/endpoints" | ||
| "github.com/aws/aws-sdk-go/service/lightsail" | ||
|
There was a problem hiding this comment. why is lightsail brought in here? I see it added to the main struct as well, but no use for it as far as I can tell. |
||
| "github.com/aws/aws-sdk-go/service/lightsail/lightsailiface" | ||
| "go.uber.org/multierr" | ||
| "math" | ||
| "sort" | ||
| "strconv" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/ec2" | ||
| "github.com/aws/aws-sdk-go/service/ec2/ec2iface" | ||
|
|
@@ -25,19 +28,26 @@ const ( | |
|
|
||
| // EC2Pricing is the public struct to interface with AWS pricing APIs | ||
| type EC2Pricing struct { | ||
| PricingClient pricingiface.PricingAPI | ||
|
There was a problem hiding this comment. To fix https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-pelong.html#pe-endpoint , I believe we can just copy the session and change the region to always use There was a problem hiding this comment. Yes, I realized this. Will make the change and other changes. |
||
| EC2Client ec2iface.EC2API | ||
| LightsailClient lightsailiface.LightsailAPI | ||
|
|
||
| AWSSession *session.Session | ||
| onDemandCache map[string]float64 | ||
| spotCache map[string]map[string][]spotPricingEntry | ||
| lastOnDemandCachedUTC *time.Time // Updated on successful cache write | ||
| lastSpotCachedUTC *time.Time // Updated on successful cache write | ||
| } | ||
|
|
||
| // EC2PricingIface is the EC2Pricing interface mainly used to mock out ec2pricing during testing | ||
| type EC2PricingIface interface { | ||
| GetOndemandInstanceTypeCost(instanceType string) (float64, error) | ||
| GetSpotInstanceTypeNDayAvgCost(instanceType string, availabilityZones []string, days int) (float64, error) | ||
| // Keep hydrate functions thread safe by keeping different write data points | ||
| // In simple words, make sure they don't write the same variable/file/row etc. which they don't (they have different cache maps) | ||
| HydrateOndemandCache() error | ||
| HydrateSpotCache(days int) error | ||
| LastOnDemandCachedUTC() *time.Time | ||
|
There was a problem hiding this comment. can you change to |
||
| LastSpotCachedUTC() *time.Time | ||
| } | ||
|
|
||
| type spotPricingEntry struct { | ||
|
|
@@ -48,12 +58,27 @@ type spotPricingEntry struct { | |
| // New creates an instance of instance-selector EC2Pricing | ||
| func New(sess *session.Session) *EC2Pricing { | ||
| return &EC2Pricing{ | ||
| PricingClient: pricing.New(sess), | ||
| EC2Client: ec2.New(sess), | ||
| LightsailClient: lightsail.New(sess), | ||
| AWSSession: sess, | ||
| lastOnDemandCachedUTC: nil, | ||
| lastSpotCachedUTC: nil, | ||
| } | ||
| } | ||
|
|
||
| // LastOnDemandCachedUTC returns the UTC timestamp when the onDemandCache was last refreshed | ||
| // Returns nil if the onDemandCache has not been initialized | ||
| func (p *EC2Pricing) LastOnDemandCachedUTC() *time.Time { | ||
| return p.lastOnDemandCachedUTC | ||
| } | ||
|
|
||
| // LastSpotCachedUTC returns the UTC timestamp when the spotCache was last refreshed | ||
| // Returns nil if the spotCache has not been initialized | ||
| func (p *EC2Pricing) LastSpotCachedUTC() *time.Time { | ||
| return p.lastSpotCachedUTC | ||
| } | ||
|
|
||
| // GetSpotInstanceTypeNDayAvgCost retrieves the spot price history for a given AZ from the past N days and averages the price | ||
| // Passing an empty list for availabilityZones will retrieve avg cost for all AZs in the current AWSSession's region | ||
| func (p *EC2Pricing) GetSpotInstanceTypeNDayAvgCost(instanceType string, availabilityZones []string, days int) (float64, error) { | ||
|
|
@@ -66,28 +91,31 @@ func (p *EC2Pricing) GetSpotInstanceTypeNDayAvgCost(instanceType string, availab | |
| EndTime: &endTime, | ||
| InstanceTypes: []*string{&instanceType}, | ||
| } | ||
| zoneToPriceEntries := make(map[string][]spotPricingEntry) | ||
|
|
||
| if _, ok := p.spotCache[instanceType]; !ok { | ||
| var processingErr error | ||
| errAPI := p.EC2Client.DescribeSpotPriceHistoryPages(&spotPriceHistInput, func(dspho *ec2.DescribeSpotPriceHistoryOutput, b bool) bool { | ||
| for _, history := range dspho.SpotPriceHistory { | ||
| var spotPrice float64 | ||
| spotPrice, errParse := strconv.ParseFloat(*history.SpotPrice, 64) | ||
| if errParse != nil { | ||
| processingErr = multierr.Append(processingErr, errParse) | ||
| continue | ||
| } | ||
| zone := *history.AvailabilityZone | ||
| zoneToPriceEntries[zone] = append(zoneToPriceEntries[zone], spotPricingEntry{ | ||
| Timestamp: *history.Timestamp, | ||
| SpotPrice: spotPrice, | ||
| }) | ||
| } | ||
| return true | ||
| }) | ||
| if errAPI != nil { | ||
| return float64(-1), errAPI | ||
| } | ||
| if processingErr != nil { | ||
| return float64(-1), processingErr | ||
| } | ||
| } else { | ||
| for zone, priceEntries := range p.spotCache[instanceType] { | ||
|
|
@@ -112,7 +140,7 @@ func (p *EC2Pricing) GetSpotInstanceTypeNDayAvgCost(instanceType string, availab | |
| aggregateZonePriceSum += p.calculateSpotAggregate(priceEntries) | ||
| } | ||
|
|
||
| return aggregateZonePriceSum / float64(numOfZones), nil | ||
| } | ||
|
|
||
| func (p *EC2Pricing) calculateSpotAggregate(spotPriceEntries []spotPricingEntry) float64 { | ||
|
|
@@ -133,11 +161,16 @@ func (p *EC2Pricing) calculateSpotAggregate(spotPriceEntries []spotPricingEntry) | |
| duration := spotPriceEntries[int(math.Max(float64(i-1), 0))].Timestamp.Sub(entry.Timestamp).Minutes() | ||
| priceSum += duration * entry.SpotPrice | ||
| } | ||
| return priceSum / totalDuration | ||
| } | ||
|
|
||
| // GetOndemandInstanceTypeCost retrieves the on-demand hourly cost for the specified instance type | ||
| func (p *EC2Pricing) GetOndemandInstanceTypeCost(instanceType string) (float64, error) { | ||
| // Check cache first and return it if available | ||
| if price, ok := p.onDemandCache[instanceType]; ok { | ||
| return price, nil | ||
| } | ||
|
|
||
| regionDescription := p.getRegionForPricingAPI() | ||
| // TODO: mac.metal instances cannot be found with the below filters | ||
| productInput := pricing.GetProductsInput{ | ||
|
|
@@ -153,25 +186,27 @@ func (p *EC2Pricing) GetOndemandInstanceTypeCost(instanceType string) (float64, | |
| }, | ||
| } | ||
|
|
||
| pricePerUnitInUSD := float64(-1) | ||
| var processingErr error | ||
| // FIXME Only works for us-east-1 and ap-south-1 | ||
| // https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-pelong.html#pe-endpoint | ||
| errAPI := p.PricingClient.GetProductsPages(&productInput, func(pricingOutput *pricing.GetProductsOutput, nextPage bool) bool { | ||
| var errParse error | ||
| for _, priceDoc := range pricingOutput.PriceList { | ||
| _, pricePerUnitInUSD, errParse = parseOndemandUnitPrice(priceDoc) | ||
| } | ||
| if errParse != nil { | ||
| processingErr = multierr.Append(processingErr, errParse) | ||
| // keep going through pages if we can't parse the pricing doc | ||
| return true | ||
| } | ||
| return false | ||
| }) | ||
| if errAPI != nil { | ||
| return -1, errAPI | ||
| } | ||
| if processingErr != nil { | ||
| return -1, processingErr | ||
| } | ||
| return pricePerUnitInUSD, nil | ||
| } | ||
|
|
@@ -181,7 +216,7 @@ func (p *EC2Pricing) GetOndemandInstanceTypeCost(instanceType string) (float64, | |
| // There is no TTL on cache entries | ||
| // You'll only want to use this if you don't mind a long startup time (around 30 seconds) and will query the cache often after that. | ||
| func (p *EC2Pricing) HydrateSpotCache(days int) error { | ||
| newCache := make(map[string]map[string][]spotPricingEntry) | ||
|
|
||
| endTime := time.Now().UTC() | ||
| startTime := endTime.Add(time.Hour * time.Duration(24*-1*days)) | ||
|
|
@@ -191,14 +226,17 @@ func (p *EC2Pricing) HydrateSpotCache(days int) error { | |
| EndTime: &endTime, | ||
| } | ||
| var processingErr error | ||
| errAPI := p.EC2Client.DescribeSpotPriceHistoryPages(&spotPriceHistInput, func(dspho *ec2.DescribeSpotPriceHistoryOutput, b bool) bool { | ||
| for _, history := range dspho.SpotPriceHistory { | ||
| spotPrice, errFloat := strconv.ParseFloat(*history.SpotPrice, 64) | ||
| if errFloat != nil { | ||
| processingErr = multierr.Append(processingErr, errFloat) | ||
| continue | ||
| } | ||
| instanceType := *history.InstanceType | ||
| zone := *history.AvailabilityZone | ||
| if _, ok := newCache[instanceType]; !ok { | ||
| newCache[instanceType] = make(map[string][]spotPricingEntry) | ||
| } | ||
| newCache[instanceType][zone] = append(newCache[instanceType][zone], spotPricingEntry{ | ||
| Timestamp: *history.Timestamp, | ||
|
|
@@ -207,20 +245,21 @@ func (p *EC2Pricing) HydrateSpotCache(days int) error { | |
| } | ||
| return true | ||
| }) | ||
| if errAPI != nil { | ||
| return errAPI | ||
| } | ||
| cTime := time.Now().UTC() | ||
| p.spotCache = newCache | ||
| p.lastSpotCachedUTC = &cTime | ||
| return processingErr | ||
| } | ||
|
|
||
| // HydrateOndemandCache makes a bulk request to the pricing api to retrieve all instance type pricing and stores them in a local cache | ||
| // If HydrateOndemandCache is called more than once, the cache will be fully refreshed | ||
| // There is no TTL on cache entries | ||
| func (p *EC2Pricing) HydrateOndemandCache() error { | ||
| newOnDemandCache := make(map[string]float64) | ||
|
|
||
| regionDescription := p.getRegionForPricingAPI() | ||
| productInput := pricing.GetProductsInput{ | ||
| ServiceCode: aws.String(serviceCode), | ||
|
|
@@ -233,17 +272,27 @@ func (p *EC2Pricing) HydrateOndemandCache() error { | |
| {Type: aws.String(pricing.FilterTypeTermMatch), Field: aws.String("tenancy"), Value: aws.String("shared")}, | ||
| }, | ||
| } | ||
| var processingErr error | ||
| // FIXME Only works for us-east-1 and ap-south-1 | ||
| // https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-pelong.html#pe-endpoint | ||
| errAPI := p.PricingClient.GetProductsPages(&productInput, func(pricingOutput *pricing.GetProductsOutput, nextPage bool) bool { | ||
| for _, priceDoc := range pricingOutput.PriceList { | ||
| instanceTypeName, price, errParse := parseOndemandUnitPrice(priceDoc) | ||
| if errParse != nil { | ||
| processingErr = multierr.Append(processingErr, errParse) | ||
| continue | ||
| } | ||
| newOnDemandCache[instanceTypeName] = price | ||
| } | ||
| return true | ||
| }) | ||
| if errAPI != nil { | ||
| return errAPI | ||
| } | ||
| cTime := time.Now().UTC() | ||
| p.onDemandCache = newOnDemandCache | ||
| p.lastOnDemandCachedUTC = &cTime | ||
| return processingErr | ||
| } | ||
|
|
||
| // getRegionForPricingAPI attempts to retrieve the region description based on the AWS session used to create | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
can you move these non-standard lib imports to the next stanza?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sure. Will add that to my fmt rules for this project.