aws-base-uplifts

.github/workflows/build.yml

@@ -300,7 +300,7 @@ jobs:
           --resource volumes_presented \
           --method describeVolumes \
           --parameters '{ "region": "ap-southeast-2" }' \
-          | jq -r '.line_items[].volumeId')" 
+          | jq -r '.line_items[].volume_id')" 
         matchingVolumes="$(echo "${volumeIDs}" | grep "vol-00100000000000000" )"
         if [ "${matchingVolumes}" = "" ]; then
           echo "Mocked CLI HTTP templated Test Failed with no matching buckets"
@@ -336,6 +336,9 @@ jobs:
         openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_rubbish_key.pem -out ${{ github.workspace }}/test/credentials/pg_rubbish_cert.pem  -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
 
     - name: Run mocked robot tests
+      env:
+        AWS_ACCESS_KEY_ID: dummy
+        AWS_SECRET_ACCESS_KEY: dummy
       run: |
         export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/test/python"
         robot -d test/robot/reports/mocked test/robot/cli/mocked

.vscode/launch.json

@@ -93,6 +93,44 @@
                 "{ \"region\": \"ap-southeast-2\", \"Bucket\": \"stackql-trial-bucket-02\", \"Status\": \"Disabled\", \"line_items\": [ { \"attribute_name\": \"stackql:createdBy\", \"attribute_value\": \"stackql-user\", \"attribute_value_type\": \"string\" } ] }"
             ]
         },
+        {
+            "type": "go",
+            "request": "launch",
+            "name": "CLI aws form out xml back: ec2 describe volumes",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/interrogate",
+            "envFile": "${workspaceFolder}/.vscode/.env",
+            "args": [
+                "query",
+                "--svc-file-path=${workspaceFolder}/test/registry-simple/src/aws/v0.1.0/services/ec2.yaml",
+                "--prov-file-path=${workspaceFolder}/test/registry-simple/src/aws/v0.1.0/provider.yaml",
+                "--resource",
+                "volumes_post_naively_presented",
+                "--method",
+                "describeVolumes",
+                "--parameters",
+                "{ \"region\": \"ap-southeast-2\" }"
+            ]
+        },
+        {
+            "type": "go",
+            "request": "launch",
+            "name": "CLI aws form out xml back: ec2 describe volumes",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/interrogate",
+            "envFile": "${workspaceFolder}/.vscode/.env",
+            "args": [
+                "query",
+                "--svc-file-path=${workspaceFolder}/../stackql-provider-registry/providers/src/aws/v00.00.00000/services/ec2_native_updated.yaml",
+                "--prov-file-path=${workspaceFolder}/../stackql-provider-registry/providers/src/aws/v00.00.00000/provider.yaml",
+                "--resource",
+                "vpcs",
+                "--method",
+                "DescribeVpcs",
+                "--parameters",
+                "{ \"region\": \"ap-southeast-2\" }"
+            ]
+        },
         {
             "type": "go",
             "request": "launch",

anysdk/http_armoury_params.go

@@ -135,11 +135,17 @@ func (hap *standardHTTPArmouryParameters) SetNextPage(
 		tokenName := tokenKey.GetName()
 		bm[tokenName] = token
 		er, _ := ops.GetRequest()
-		b, err := ops.MarshalBody(bm, er)
-		if err != nil {
-			return nil, err
+		marshalledBody := ops.MarshalBody(bm, er)
+		b := marshalledBody.GetBytes()
+		marshallErr, hasMarshallErr := marshalledBody.GetError()
+
+		if hasMarshallErr {
+			return nil, marshallErr
+		}
+
+		if len(b) > 0 {
+			rv.Body = io.NopCloser(bytes.NewBuffer(b))
 		}
-		rv.Body = io.NopCloser(bytes.NewBuffer(b))
 		rv.ContentLength = int64(len(b))
 		return rv, nil
 	default:

anysdk/operation_store.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/getkin/kin-openapi/openapi3filter"
+	"github.com/stackql/any-sdk/pkg/dto"
 	"github.com/stackql/any-sdk/pkg/fuzzymatch"
 	"github.com/stackql/any-sdk/pkg/media"
 	"github.com/stackql/any-sdk/pkg/parametertranslate"
@@ -95,7 +96,7 @@ type OperationStore interface {
 	GetParameter(paramKey string) (Addressable, bool)
 	GetUnionRequiredParameters() (map[string]Addressable, error)
 	GetPaginationResponseTokenSemantic() (TokenSemantic, bool)
-	MarshalBody(body interface{}, expectedRequest ExpectedRequest) ([]byte, error)
+	MarshalBody(body interface{}, expectedRequest ExpectedRequest) dto.MarshalledBody
 	GetRequestBodySchema() (Schema, error)
 	GetNonBodyParameters() map[string]Addressable
 	GetRequestBodyAttributesNoRename() (map[string]Addressable, error)
@@ -1322,32 +1323,35 @@ func selectServer(servers openapi3.Servers, inputParams map[string]interface{})
 
 func (op *standardOpenAPIOperationStore) acceptPathParam(mutableParamMap map[string]interface{}) {}
 
-func (op *standardOpenAPIOperationStore) MarshalBody(body interface{}, expectedRequest ExpectedRequest) ([]byte, error) {
+func (op *standardOpenAPIOperationStore) MarshalBody(body interface{}, expectedRequest ExpectedRequest) dto.MarshalledBody {
 	return op.marshalBody(body, expectedRequest)
 }
 
-func (op *standardOpenAPIOperationStore) marshalBody(body interface{}, expectedRequest ExpectedRequest) ([]byte, error) {
+func (op *standardOpenAPIOperationStore) marshalBody(body interface{}, expectedRequest ExpectedRequest) dto.MarshalledBody {
 	_, isTransform := expectedRequest.GetTransform()
 	if isTransform {
-		return op.transformRequestBodyMap(body.(map[string]interface{}))
+		b, err := op.transformRequestBodyMap(body.(map[string]interface{}))
+		return dto.NewMarshalledBody(b, err)
 	}
 	mediaType := expectedRequest.GetBodyMediaType()
 	if expectedRequest.GetSchema() != nil {
 		mediaType = expectedRequest.GetSchema().ExtractMediaTypeSynonym(mediaType)
 	}
 	switch mediaType {
 	case media.MediaTypeJson:
-		return json.Marshal(body)
+		b, err := json.Marshal(body)
+		return dto.NewMarshalledBody(b, err)
 	case media.MediaTypeXML, media.MediaTypeTextXML:
-		return xmlmap.MarshalXMLUserInput(
+		b, err := xmlmap.MarshalXMLUserInput(
 			body,
 			expectedRequest.GetFinalSchema().GetXMLALiasOrName(),
 			op.getXMLTransform(),
 			op.getXMLDeclaration(),
 			op.getXMLRootAnnotation(),
 		)
+		return dto.NewMarshalledBody(b, err)
 	}
-	return nil, fmt.Errorf("media type = '%s' not supported", expectedRequest.GetBodyMediaType())
+	return dto.NewMarshalledBody(nil, fmt.Errorf("media type = '%s' not supported", expectedRequest.GetBodyMediaType()))
 }
 
 func (op *standardOpenAPIOperationStore) parameterize(prov Provider, parentDoc Service, inputParams HttpParameters, requestBody interface{}) (*openapi3filter.RequestValidationInput, error) {
@@ -1436,16 +1440,21 @@ func (op *standardOpenAPIOperationStore) parameterize(prov Provider, parentDoc S
 	}
 	contentTypeHeaderRequired := false
 	var bodyReader io.Reader
-	predOne := !util.IsNil(requestBody)
+	// predOne := !util.IsNil(requestBody)
 	predTwo := !util.IsNil(op.Request)
-	if predOne && predTwo {
+	if predTwo {
 		// TODO: transform
-		b, err := op.marshalBody(requestBody, op.Request)
-		if err != nil {
-			return nil, err
+		marshalledBody := op.marshalBody(requestBody, op.Request)
+		b := marshalledBody.GetBytes()
+		marshalledBodyErr, hassError := marshalledBody.GetError()
+		if hassError {
+			return nil, marshalledBodyErr
+		}
+		if len(b) > 0 {
+			bodyReader = bytes.NewReader(b)
+			contentTypeHeaderRequired = true
 		}
-		bodyReader = bytes.NewReader(b)
-		contentTypeHeaderRequired = true
+
 	}
 	// TODO: clean up
 	sv = strings.TrimSuffix(sv, "/")
@@ -1641,10 +1650,13 @@ func (op *standardOpenAPIOperationStore) transformRequestBodyMap(input map[strin
 }
 
 func (op *standardOpenAPIOperationStore) transformRequestBodyBytes(input []byte) ([]byte, error) {
+	var inputStr string = ""
+	if len(input) > 0 {
+		inputStr = string(input)
+	}
 	if op.Request != nil {
 		requestTransform, requestTransformExists := op.Request.GetTransform()
 		if requestTransformExists {
-			inputStr := string(input)
 			streamTransformerFactory := stream_transform.NewStreamTransformerFactory(
 				requestTransform.GetType(),
 				requestTransform.GetBody(),

anysdk/operation_store_test.go

@@ -563,10 +563,12 @@ func TestXMLRequestBody(t *testing.T) {
 		"ChangeBatch": sqlRequestString,
 	}
 
-	processed, err := ops.MarshalBody(requestBodyMap, expectedRequest)
+	marshalledBody := ops.MarshalBody(requestBodyMap, expectedRequest)
+	processed := marshalledBody.GetBytes()
+	marshallErr, hasMarshallErr := marshalledBody.GetError()
 
-	if err != nil {
-		t.Fatalf("Test failed: %v", err)
+	if hasMarshallErr {
+		t.Fatalf("Test failed: %v", marshallErr)
 	}
 
 	expectedMatureBody := "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ChangeResourceRecordSetsRequest xmlns=\"https://route53.amazonaws.com/doc/2013-04-01/\"><ChangeBatch><Changes><Change><Action>CREATE</Action><ResourceRecordSet><Name>my.domain.com</Name><Type>A</Type><TTL>900</TTL><ResourceRecords><ResourceRecord><Value>10.10.10.10</Value></ResourceRecord></ResourceRecords></ResourceRecordSet></Change></Changes></ChangeBatch></ChangeResourceRecordSetsRequest>"
@@ -610,10 +612,12 @@ func TestJSONRequestBody(t *testing.T) {
 		"name": "my-test-bucket",
 	}
 
-	processed, err := ops.MarshalBody(requestBodyMap, expectedRequest)
+	marshalledBody := ops.MarshalBody(requestBodyMap, expectedRequest)
+	processed := marshalledBody.GetBytes()
+	marshallErr, hasMarshallErr := marshalledBody.GetError()
 
-	if err != nil {
-		t.Fatalf("Test failed: %v", err)
+	if hasMarshallErr {
+		t.Fatalf("Test failed: %v", marshallErr)
 	}
 
 	expectedMatureBody := `{"name":"my-test-bucket"}`

cicd/tools/api/exported_funcs.txt

@@ -105,6 +105,7 @@ func NewHttpPreparatorStream
 func NewInterrogator
 func NewJSONPathResolver
 func NewLocalTemplateExecutor
+func NewMarshalledBody
 func NewMethodAnalysisInput
 func NewMethodAnalyzer
 func NewNaiveBodyTranslator

cicd/tools/api/exported_interfaces.txt

@@ -58,6 +58,7 @@ type MapReader interface
 type MapStream interface
 type MapStreamCollection interface
 type MapWriter interface
+type MarshalledBody interface
 type MethodAggregateStaticAnalyzer interface
 type MethodAnalysisInput interface
 type MethodAnalysisOutput interface

docs/cli.md

@@ -123,6 +123,20 @@ build/anysdk query \
 
 ```
 
+EC2 cod dev:
+
+```bash
+
+build/anysdk query \
+  --svc-file-path="test/registry-simple/src/aws/v0.1.0/services/ec2.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="test/registry-simple/src/aws/v0.1.0/provider.yaml" \
+  --resource volumes_presented \
+  --method describeVolumes \
+  --parameters '{ "region": "ap-southeast-2" }' 
+
+```
+
 S3 one of the great challenges:
 
 

docs/development_of_ec2_and_s3.md

@@ -0,0 +1,110 @@
+
+
+## s3
+
+S3 calls:
+
+
+```bash
+
+build/anysdk query \
+  --svc-file-path="test/registry-simple/src/aws/v0.1.0/services/s3.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="test/registry-simple/src/aws/v0.1.0/provider.yaml" \
+  --resource bucket_abac \
+  --method get_bucket_abac \
+  --parameters '{ "region": "ap-southeast-1", "Bucket": "stackql-trial-bucket-01" }' 
+
+
+build/anysdk query \
+  --svc-file-path="test/registry/src/aws/v0.1.0/services/s3.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="test/registry/src/aws/v0.1.0/provider.yaml" \
+  --resource bucket_abac \
+  --method put_bucket_abac \
+  --parameters '{ "region": "ap-southeast-1", "Bucket": "stackql-trial-bucket-01", "Status": "Enabled" }' 
+
+build/anysdk query \
+  --svc-file-path="test/registry/src/aws/v0.1.0/services/s3.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="test/registry/src/aws/v0.1.0/provider.yaml" \
+  --resource bucket_abac \
+  --method get_bucket_abac \
+  --parameters '{ "region": "ap-southeast-1", "Bucket": "stackql-trial-bucket-01" }' 
+
+
+## BLEEDING EDGE
+build/anysdk query \
+  --svc-file-path="$HOME/stackql/stackql-provider-registry/providers/src/aws/v00.00.00000/services/ec2_native_updated_v2.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="$HOME/stackql/stackql-provider-registry/providers/src/aws/v00.00.00000/provider.yaml" \
+  --resource vpcs \
+  --method describe \
+  --parameters '{ "region": "ap-southeast-2" }' 
+
+build/anysdk query \
+  --svc-file-path="$HOME/stackql/stackql-provider-registry/providers/src/aws/v00.00.00000/services/ec2_native_updated_v2.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="$HOME/stackql/stackql-provider-registry/providers/src/aws/v00.00.00000/provider.yaml" \
+  --resource subnets \
+  --method describe \
+  --parameters '{ "region": "ap-southeast-2" }' 
+
+```
+
+## ec2
+
+
+ec2 calls:
+
+
+```bash
+
+build/anysdk query \
+  --svc-file-path="test/registry-simple/src/aws/v0.1.0/services/ec2.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="test/registry-simple/src/aws/v0.1.0/provider.yaml" \
+  --resource volumes_naively_presented \
+  --method describeVolumes \
+  --parameters '{ "region": "ap-southeast-2" }' 
+
+
+build/anysdk query \
+  --svc-file-path="test/registry-simple/src/aws/v0.1.0/services/ec2.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="test/registry-simple/src/aws/v0.1.0/provider.yaml" \
+  --resource volumes_post_naively_presented \
+  --method describeVolumes \
+  --parameters '{ "region": "ap-southeast-2" }' 
+
+```
+
+Regression tests:
+
+```bash
+
+build/anysdk query \
+    --svc-file-path="test/registry/src/aws/v0.1.0/services/ec2.yaml" \
+    --tls.allowInsecure \
+    --prov-file-path="test/registry/src/aws/v0.1.0/provider.yaml" \
+    --resource volumes_presented \
+    --method describeVolumes \
+    --parameters '{ "region": "ap-southeast-2" }' | jq -r '.line_items[].volume_id'
+
+build/anysdk query \
+  --svc-file-path="test/registry/src/aws/v0.1.0/services/ec2.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="test/registry/src/aws/v0.1.0/provider.yaml" \
+  --resource volumes_post_naively_presented \
+  --method describeVolumes \
+  --parameters '{ "region": "ap-southeast-2" }' 
+
+build/anysdk query \
+  --svc-file-path="test/registry-mocked/src/aws/v0.1.0/services/ec2.yaml" \
+  --tls.allowInsecure \
+  --prov-file-path="test/registry-mocked/src/aws/v0.1.0/provider.yaml" \
+  --resource volumes_post_naively_presented \
+  --method describeVolumes \
+  --parameters '{ "region": "ap-southeast-2" }' 
+
+```

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/stretchr/testify v1.10.0
 	github.com/xo/dburl v0.23.2
 	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
+	golang.org/x/mod v0.22.0
 	golang.org/x/oauth2 v0.26.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -107,7 +108,6 @@ require (
 	go.opentelemetry.io/otel v1.35.0 // indirect
 	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	golang.org/x/crypto v0.32.0 // indirect
-	golang.org/x/mod v0.22.0 // indirect
 	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/sys v0.30.0 // indirect