diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 923729d..576b556 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,6 +97,11 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v2 + - name: Setup system dependencies + run: | + sudo apt-get update + sudo apt-get install -y jq + - name: Setup Python uses: actions/setup-python@v5.0.0 with: @@ -137,8 +142,6 @@ jobs: - name: Trivial CLI Test if: success() run: | - sudo apt-get update - sudo apt-get install -y jq result="$(build/anysdk const | jq -r '.ExtensionKeyAlwaysRequired')" if [ "$result" != "x-alwaysRequired" ]; then echo "Trivial CLI Test Failed with unexpected result: $result" @@ -189,6 +192,46 @@ jobs: else echo "Core Test passed with matching buckets: $matchingBuckets" fi + + - name: Run local templated openssl mutate test + run: | + rm -rf test/tmp/*.pem + ${{ github.workspace }}/build/anysdk query \ + --svc-file-path="test/registry/src/local_openssl/v0.1.0/services/keys.yaml" \ + --prov-file-path="test/registry/src/local_openssl/v0.1.0/provider.yaml" \ + --resource rsa \ + --method create_key_pair \ + --parameters '{ + "config_file": "test/openssl/openssl.cnf", + "key_out_file": "test/tmp/key.pem", + "cert_out_file": "test/tmp/cert.pem", + "days": 90 + }' + endDateFound="$(openssl x509 -in test/tmp/cert.pem -noout -dates | grep "notAfter")" + if [ "${endDateFound}" = "" ]; then + echo "Core Test Failed with no matching end date" + exit 1 + else + echo "Core Test passed with matching end date info: $endDateFound" + fi + + - name: Run local templated openssl select test + run: | + response="$(${{ github.workspace }}/build/anysdk query \ + --svc-file-path='test/registry/src/local_openssl/v0.1.0/services/keys.yaml' \ + --prov-file-path='test/registry/src/local_openssl/v0.1.0/provider.yaml' \ + --resource x509 \ + --method describe_certificate \ + --parameters '{ + "cert_file": "test/tmp/cert.pem" + }')" + publicKeyAlgorithm="$(echo "$response" | jq -r '.public_key_algorithm')" + if [ "${publicKeyAlgorithm}" != "rsaEncryption" ]; then + echo "Core Test Failed with unexpected public key algorithm '$publicKeyAlgorithm'" + exit 1 + else + echo "Core Test passed with matching public key algorithm: '$publicKeyAlgorithm'" + fi macosbuild: name: MacOS Build diff --git a/.vscode/.gitignore b/.vscode/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/.vscode/.gitignore @@ -0,0 +1 @@ +.env diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..aa6f420 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,54 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "go", + "request": "launch", + "name": "Run standalone file", + "mode": "auto", + "program": "${file}", + "envFile": "${workspaceFolder}/.vscode/.env", + }, + { + "type": "go", + "request": "launch", + "name": "CLI local: rsa openssl create", + "mode": "auto", + "program": "${workspaceFolder}/cmd/interrogate", + "envFile": "${workspaceFolder}/.vscode/.env", + "args": [ + "query", + "--svc-file-path=${workspaceFolder}/test/registry/src/local_openssl/v0.1.0/services/keys.yaml", + "--prov-file-path=${workspaceFolder}/test/registry/src/local_openssl/v0.1.0/provider.yaml", + "--resource", + "rsa", + "--method", + "create_key_pair", + "--parameters", + "{ \"config_file\": \"${workspaceFolder}/test/openssl/openssl.cnf\", \"key_out_file\": \"${workspaceFolder}/test/tmp/key.pem\", \"cert_out_file\": \"${workspaceFolder}/test/tmp/cert.pem\", \"days\": 90}" + ] + }, + { + "type": "go", + "request": "launch", + "name": "CLI local: x509 openssl describe", + "mode": "auto", + "program": "${workspaceFolder}/cmd/interrogate", + "envFile": "${workspaceFolder}/.vscode/.env", + "args": [ + "query", + "--svc-file-path=${workspaceFolder}/test/registry/src/local_openssl/v0.1.0/services/keys.yaml", + "--prov-file-path=${workspaceFolder}/test/registry/src/local_openssl/v0.1.0/provider.yaml", + "--resource", + "x509", + "--method", + "describe_certificate", + "--parameters", + "{ \"cert_file\": \"${workspaceFolder}/test/tmp/cert.pem\"}" + ] + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 7526e44..93c70aa 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,15 @@ A golang library to support: From those who brought you -[![StackQL](https://stackql.io/img/stackql-banner.png)](https://stackql.io/) +[![StackQL](https://stackql.io/img/stackql-banner.png)](https://stackql.io/) + +## Evolution to protocol agnostic + +The basic idea is a full decouple and abstraction of the interface from openapi. + +Based upon the fact that [golang text templates are Turing complete](https://linuxtut.com/en/2072207ec0565a80d2b2/), as are numerous other DSLs, we can use these to define and route SQL input to arbitrary interfaces. For instance, here is [the brainf@$& interpreter](https://github.com/Syuparn/go-template-bf-interpreter/blob/1b7f6a3720295c93ffa99b58a81f153bd8d7ecc8/bf-interpreter.tpl) described in the article. + +As a side note, [this review piece](https://solutionspace.blog/2021/12/04/every-simple-language-will-eventually-end-up-turing-complete/) gives some insight into whay all DSLs wind up Turing complete and also therefore rather messy for narrowband purposes. ## Acknowledgements diff --git a/anysdk/client.go b/anysdk/client.go new file mode 100644 index 0000000..a0b9d60 --- /dev/null +++ b/anysdk/client.go @@ -0,0 +1,523 @@ +package anysdk + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/stackql/any-sdk/pkg/auth_util" + "github.com/stackql/any-sdk/pkg/client" + "github.com/stackql/any-sdk/pkg/dto" + "github.com/stackql/any-sdk/pkg/internaldto" + "github.com/stackql/any-sdk/pkg/netutils" + "github.com/stackql/any-sdk/pkg/requesttranslate" +) + +var ( + _ client.AnySdkClientConfigurator = &anySdkHTTPClientConfigurator{} + _ client.AnySdkResponse = &anySdkHttpResponse{} + _ client.AnySdkClient = &anySdkHttpClient{} +) + +type anySdkHttpClient struct { + client *http.Client +} + +func newAnySdkHttpClient(client *http.Client) client.AnySdkClient { + return &anySdkHttpClient{ + client: client, + } +} + +func NewwHTTPAnySdkArgList(req *http.Request) client.AnySdkArgList { + return &anySdkArgList{ + args: []client.AnySdkArg{ + newAnySdkHTTPArg(req), + }, + } +} + +func NewAnySdkArgList(argList any) (client.AnySdkArgList, error) { + switch argList := argList.(type) { + case *http.Request: + req := argList + return NewwHTTPAnySdkArgList(req), nil + default: + return nil, fmt.Errorf("could not cast argList of type '%T' to *http.Request", argList) + } +} + +type anySdkHttpResponse struct { + reponse *http.Response +} + +func (hr *anySdkHttpResponse) IsErroneous() bool { + return hr.reponse.StatusCode >= 400 +} + +func (hr *anySdkHttpResponse) GetHttpResponse() (*http.Response, error) { + return hr.reponse, nil +} + +func newAnySdkHttpReponse(httpResponse *http.Response) client.AnySdkResponse { + return &anySdkHttpResponse{ + reponse: httpResponse, + } +} + +type anySdkArgList struct { + args []client.AnySdkArg + protocolType client.ClientProtocolType +} + +func (al *anySdkArgList) GetArgs() []client.AnySdkArg { + return al.args +} + +func (al *anySdkArgList) GetProtocolType() client.ClientProtocolType { + return al.protocolType +} + +func newAnySdkArgList(protocolType client.ClientProtocolType, args ...client.AnySdkArg) client.AnySdkArgList { + return &anySdkArgList{ + args: args, + protocolType: protocolType, + } +} + +type anySdkHTTPArg struct { + arg *http.Request +} + +func (ha *anySdkHTTPArg) GetArg() (interface{}, bool) { + precursor := ha.arg + isNil := ha.arg == nil + if isNil { + return nil, false + } + return precursor.Clone( + precursor.Context(), + ), true +} + +func newAnySdkHTTPArg(arg *http.Request) client.AnySdkArg { + return &anySdkHTTPArg{ + arg: arg, + } +} + +func (hc *anySdkHttpClient) Do(designation client.AnySdkDesignation, argList client.AnySdkArgList) (client.AnySdkResponse, error) { + firstArg := argList.GetArgs()[0] + arg, hasFirstArg := firstArg.GetArg() + if !hasFirstArg { + return nil, fmt.Errorf("could not get first argument") + } + httpReq, isHttpRequest := arg.(*http.Request) + if !isHttpRequest { + return nil, fmt.Errorf("could not cast first argument to http.Request") + } + httpResponse, httpResponseErr := hc.client.Do(httpReq) + if httpResponseErr != nil { + return nil, httpResponseErr + } + anySdkHttpResponse := newAnySdkHttpReponse(httpResponse) + return anySdkHttpResponse, nil +} + +type anySdkHTTPClientConfigurator struct { + runtimeCtx dto.RuntimeCtx + authUtil auth_util.AuthUtility + providerName string +} + +func NewAnySdkClientConfigurator( + rtCtx dto.RuntimeCtx, + provName string, +) client.AnySdkClientConfigurator { + return &anySdkHTTPClientConfigurator{ + runtimeCtx: rtCtx, + authUtil: auth_util.NewAuthUtility(), + providerName: provName, + } +} + +func (cc *anySdkHTTPClientConfigurator) InferAuthType(authCtx dto.AuthCtx, authTypeRequested string) string { + return cc.inferAuthType(authCtx, authTypeRequested) +} + +func (cc *anySdkHTTPClientConfigurator) inferAuthType(authCtx dto.AuthCtx, authTypeRequested string) string { + ft := strings.ToLower(authTypeRequested) + switch ft { + case dto.AuthAzureDefaultStr: + return dto.AuthAzureDefaultStr + case dto.AuthAPIKeyStr: + return dto.AuthAPIKeyStr + case dto.AuthBasicStr: + return dto.AuthBasicStr + case dto.AuthBearerStr: + return dto.AuthBearerStr + case dto.AuthServiceAccountStr: + return dto.AuthServiceAccountStr + case dto.AuthInteractiveStr: + return dto.AuthInteractiveStr + case dto.AuthNullStr: + return dto.AuthNullStr + case dto.AuthAWSSigningv4Str: + return dto.AuthAWSSigningv4Str + case dto.AuthCustomStr: + return dto.AuthCustomStr + case dto.OAuth2Str: + return dto.OAuth2Str + } + if authCtx.KeyFilePath != "" || authCtx.KeyEnvVar != "" { + return dto.AuthServiceAccountStr + } + return dto.AuthNullStr +} + +func (cc *anySdkHTTPClientConfigurator) Auth( + authCtx *dto.AuthCtx, + authTypeRequested string, + enforceRevokeFirst bool, +) (client.AnySdkClient, error) { + authCtx = authCtx.Clone() + at := cc.inferAuthType(*authCtx, authTypeRequested) + switch at { + case dto.AuthAPIKeyStr: + httpClient, httpClientErr := cc.authUtil.ApiTokenAuth(authCtx, cc.runtimeCtx, false) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + case dto.AuthBearerStr: + httpClient, httpClientErr := cc.authUtil.ApiTokenAuth(authCtx, cc.runtimeCtx, true) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + case dto.AuthServiceAccountStr: + scopes := authCtx.Scopes + httpClient, httpClientErr := cc.authUtil.GoogleOauthServiceAccount(cc.providerName, authCtx, scopes, cc.runtimeCtx) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + case dto.OAuth2Str: + if authCtx.GrantType == dto.ClientCredentialsStr { + scopes := authCtx.Scopes + httpClient, httpClientErr := cc.authUtil.GenericOauthClientCredentials(authCtx, scopes, cc.runtimeCtx) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + } + case dto.AuthBasicStr: + httpClient, httpClientErr := cc.authUtil.BasicAuth(authCtx, cc.runtimeCtx) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + case dto.AuthCustomStr: + httpClient, httpClientErr := cc.authUtil.CustomAuth(authCtx, cc.runtimeCtx) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + case dto.AuthAzureDefaultStr: + httpClient, httpClientErr := cc.authUtil.AzureDefaultAuth(authCtx, cc.runtimeCtx) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + case dto.AuthInteractiveStr: + httpClient, httpClientErr := cc.authUtil.GCloudOAuth(cc.runtimeCtx, authCtx, enforceRevokeFirst) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + case dto.AuthAWSSigningv4Str: + httpClient, httpClientErr := cc.authUtil.AwsSigningAuth(authCtx, cc.runtimeCtx) + if httpClientErr != nil { + return nil, httpClientErr + } + return newAnySdkHttpClient(httpClient), nil + case dto.AuthNullStr: + httpClient := netutils.GetHTTPClient(cc.runtimeCtx, http.DefaultClient) + return newAnySdkHttpClient(httpClient), nil + } + return nil, fmt.Errorf("could not infer auth type") +} + +//nolint:nestif,mnd // acceptable for now +func parseReponseBodyIfErroneous(response *http.Response) (string, error) { + if response != nil { + if response.StatusCode >= 300 { + if response.Body != nil { + bodyBytes, bErr := io.ReadAll(response.Body) + if bErr != nil { + return "", bErr + } + bodyStr := string(bodyBytes) + response.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) + if len(bodyStr) > 0 { + return fmt.Sprintf("http response status code: %d, response body: %s", response.StatusCode, bodyStr), nil + } + } + return fmt.Sprintf("http response status code: %d, response body is nil", response.StatusCode), nil + } + } + return "", nil +} + +//nolint:nestif // acceptable for now +func parseReponseBodyIfPresent(response *http.Response) (string, error) { + if response != nil { + if response.Body != nil { + bodyBytes, bErr := io.ReadAll(response.Body) + if bErr != nil { + return "", bErr + } + bodyStr := string(bodyBytes) + response.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) + if len(bodyStr) > 0 { + return fmt.Sprintf("http response status code: %d, response body: %s", response.StatusCode, bodyStr), nil + } + return fmt.Sprintf("http response status code: %d, response body is nil", response.StatusCode), nil + } + } + return "nil response", nil +} + +type httpClientConfiguratorInput struct { + authCtx *dto.AuthCtx + authType string + enforceRevokeFirst bool +} + +func NewHttpClientConfiguratorInput( + authCtx *dto.AuthCtx, + authType string, + enforceRevokeFirst bool, +) client.ClientConfiguratorInput { + return &httpClientConfiguratorInput{ + authCtx: authCtx, + authType: authType, + enforceRevokeFirst: enforceRevokeFirst, + } +} + +func (hci *httpClientConfiguratorInput) GetAuthContext() *dto.AuthCtx { + return hci.authCtx +} + +func (hci *httpClientConfiguratorInput) GetAuthType() string { + return hci.authType +} + +func (hci *httpClientConfiguratorInput) GetEnforceRevokeFirst() bool { + return hci.enforceRevokeFirst +} + +type anySdkHTTPDesignation struct { + method OperationStore +} + +func NewAnySdkOpStoreDesignation(method OperationStore) client.AnySdkDesignation { + return newAnySdkOpStoreDesignation(method) +} + +func newAnySdkOpStoreDesignation(method OperationStore) client.AnySdkDesignation { + return &anySdkHTTPDesignation{ + method: method, + } +} + +func (hd *anySdkHTTPDesignation) GetDesignation() (interface{}, bool) { + return hd.method, hd.method != nil +} + +func inferMaxResultsElement(OperationStore) internaldto.HTTPElement { + return internaldto.NewHTTPElement( + internaldto.QueryParam, + "maxResults", + ) +} + +func HTTPApiCallFromRequest( + cc client.AnySdkClientConfigurator, + runtimeCtx dto.RuntimeCtx, + authCtx *dto.AuthCtx, + authTypeRequested string, + enforceRevokeFirst bool, + outErrFile io.Writer, + prov Provider, + method OperationStore, + request *http.Request, +) (*http.Response, error) { + return httpApiCallFromRequest( + cc, + runtimeCtx, + authCtx, + authTypeRequested, + enforceRevokeFirst, + outErrFile, + method, + request, + ) +} + +func GetMonitorRequest(urlStr string) (client.AnySdkArgList, error) { + urlObj, err := url.Parse(urlStr) + if err != nil { + return nil, err + } + if strings.ToLower(urlObj.Scheme) != "http" && strings.ToLower(urlObj.Scheme) != "https" { + return nil, fmt.Errorf("url scheme '%s' disallowed; must be http or https", urlObj.Scheme) + } + req, err := http.NewRequest( + http.MethodGet, + urlStr, + nil, + ) + if err != nil { + return nil, err + } + req = req.WithContext(context.Background()) + return newAnySdkArgList( + client.HTTP, + newAnySdkHTTPArg(req), + ), nil +} + +func CallFromSignature( + cc client.AnySdkClientConfigurator, + runtimeCtx dto.RuntimeCtx, + authCtx *dto.AuthCtx, + authTypeRequested string, + enforceRevokeFirst bool, + outErrFile io.Writer, + prov Provider, + designation client.AnySdkDesignation, + argList client.AnySdkArgList, +) (client.AnySdkResponse, error) { + rawDesignation, hasRawDesignation := designation.GetDesignation() + if !hasRawDesignation { + return nil, fmt.Errorf("could not get raw designation") + } + switch castRawDesignation := rawDesignation.(type) { + case OperationStore: + method := castRawDesignation + firstArg := argList.GetArgs()[0] + arg, hasFirstArg := firstArg.GetArg() + if !hasFirstArg { + return nil, fmt.Errorf("could not get first argument") + } + httpReq, isHttpRequest := arg.(*http.Request) + if !isHttpRequest { + return nil, fmt.Errorf("could not cast first argument to http.Request") + } + httpResponse, httpResponseErr := httpApiCallFromRequest( + cc, + runtimeCtx, + authCtx, + authTypeRequested, + enforceRevokeFirst, + outErrFile, + method, + httpReq.Clone( + httpReq.Context(), + ), + ) + if httpResponseErr != nil { + return nil, httpResponseErr + } + anySdkHttpResponse := newAnySdkHttpReponse(httpResponse) + return anySdkHttpResponse, nil + default: + return nil, fmt.Errorf("could not cast designation of type '%T' to OperationStore", designation) + } +} + +func httpApiCallFromRequest( + cc client.AnySdkClientConfigurator, + runtimeCtx dto.RuntimeCtx, + authCtx *dto.AuthCtx, + authTypeRequested string, + enforceRevokeFirst bool, + outErrFile io.Writer, + method OperationStore, + request *http.Request, +) (*http.Response, error) { + httpClient, httpClientErr := cc.Auth(authCtx, authTypeRequested, enforceRevokeFirst) + if httpClientErr != nil { + return nil, httpClientErr + } + request.Header.Del("Authorization") + requestTranslator, err := requesttranslate.NewRequestTranslator(method.GetRequestTranslateAlgorithm()) + if err != nil { + return nil, err + } + translatedRequest, err := requestTranslator.Translate(request) + if err != nil { + return nil, err + } + if runtimeCtx.HTTPLogEnabled { + urlStr := "" + methodStr := "" + if translatedRequest != nil && translatedRequest.URL != nil { + urlStr = translatedRequest.URL.String() + methodStr = translatedRequest.Method + } + //nolint:errcheck // output stream + outErrFile.Write([]byte(fmt.Sprintf("http request url: '%s', method: '%s'\n", urlStr, methodStr))) + body := translatedRequest.Body + if body != nil { + b, bErr := io.ReadAll(body) + if bErr != nil { + //nolint:errcheck // output stream + outErrFile.Write([]byte(fmt.Sprintf("error inpecting http request body: %s\n", bErr.Error()))) + } + bodyStr := string(b) + translatedRequest.Body = io.NopCloser(bytes.NewBuffer(b)) + //nolint:errcheck // output stream + outErrFile.Write([]byte(fmt.Sprintf("http request body = '%s'\n", bodyStr))) + } + } + r, err := httpClient.Do( + newAnySdkOpStoreDesignation(method), + newAnySdkArgList( + client.HTTP, + newAnySdkHTTPArg(translatedRequest), + ), + ) + if err != nil { + return nil, err + } + httpResponse, _ := r.GetHttpResponse() + responseErrorBodyToPublish, reponseParseErr := parseReponseBodyIfErroneous(httpResponse) + if reponseParseErr != nil { + return nil, reponseParseErr + } + if responseErrorBodyToPublish != "" { + //nolint:errcheck // output stream + outErrFile.Write([]byte(fmt.Sprintf("%s\n", responseErrorBodyToPublish))) + } else if runtimeCtx.HTTPLogEnabled { + reponseBodyStr, _ := parseReponseBodyIfPresent(httpResponse) + //nolint:errcheck // output stream + outErrFile.Write([]byte(fmt.Sprintf("%s\n", reponseBodyStr))) + } + if err != nil { + if runtimeCtx.HTTPLogEnabled { + //nolint:errcheck // output stream + outErrFile.Write([]byte( + fmt.Sprintln(fmt.Sprintf("http response error: %s", err.Error()))), //nolint:gosimple,lll // TODO: sweep through this sort of nonsense + ) + } + return nil, err + } + return httpResponse, err +} diff --git a/anysdk/client_test.go b/anysdk/client_test.go new file mode 100644 index 0000000..8fcf55a --- /dev/null +++ b/anysdk/client_test.go @@ -0,0 +1,78 @@ +package anysdk_test + +import ( + "os" + "path" + "testing" + + . "github.com/stackql/any-sdk/anysdk" + "github.com/stackql/any-sdk/pkg/fileutil" + + "gotest.tools/assert" + + "github.com/stackql/any-sdk/pkg/local_template_executor" +) + +var ( + testRoot string +) + +func init() { + var err error + OpenapiFileRoot, err = fileutil.GetFilePathFromRepositoryRoot("test/registry/src") + if err != nil { + os.Exit(1) + } + testRoot, err = fileutil.GetFilePathFromRepositoryRoot("test") + if err != nil { + os.Exit(1) + } +} + +func TestLocalTemplateClient(t *testing.T) { + providerPath := path.Join(OpenapiFileRoot, "local_openssl", "v0.1.0", "provider.yaml") + servicePath := path.Join(OpenapiFileRoot, "local_openssl", "v0.1.0", "services", "keys.yaml") + pb, err := os.ReadFile(providerPath) + if err != nil { + t.Fatalf("error loading provider doc: %v", err) + } + prov, err := LoadProviderDocFromBytes(pb) + if err != nil { + t.Fatalf("error loading provider doc: %v", err) + } + assert.Assert(t, prov != nil) + svc, err := LoadProviderAndServiceFromPaths(providerPath, servicePath) + if err != nil { + t.Fatalf("error loading service: %v", err) + } + res, err := svc.GetResource("rsa") + if err != nil { + t.Fatalf("error loading resource: %v", err) + } + opStore, err := res.FindMethod("create_key_pair") + if err != nil { + t.Fatalf("error loading method: %v", err) + } + assert.Assert(t, opStore != nil) + args := opStore.GetInline() + if len(args) == 0 { + t.Fatalf("no args found") + } + executor := local_template_executor.NewLocalTemplateExecutor(args[0], args[1:], nil) + resp, err := executor.Execute(map[string]any{ + "parameters": map[string]any{ + "config_file": path.Join(testRoot, "openssl/openssl.cnf"), + "key_out_file": path.Join(testRoot, "tmp/key.pem"), + "cert_out_file": path.Join(testRoot, "tmp/cert.pem"), + "days": 90, + }, + }) + if err != nil { + t.Fatalf("error executing command: %v", err) + } + stdOut, ok := resp.GetStdOut() + if !ok { + t.Fatalf("no stdout") + } + t.Logf("stdout: %s", stdOut.String()) +} diff --git a/anysdk/concerted_action.go b/anysdk/concerted_action.go new file mode 100644 index 0000000..55e0945 --- /dev/null +++ b/anysdk/concerted_action.go @@ -0,0 +1,195 @@ +package anysdk + +import ( + "fmt" +) + +type MethodAnalysisInput interface { + GetService() Service + GetMethod() OperationStore + IsNilResponseAllowed() bool + GetColumns() []ColumnDescriptor +} + +type standardMethodAnalysisInput struct { + method OperationStore + service Service + isNilResponseAllowed bool + columns []ColumnDescriptor +} + +func NewMethodAnalysisInput( + method OperationStore, + service Service, + isNilResponseAllowed bool, + columns []ColumnDescriptor, +) MethodAnalysisInput { + return &standardMethodAnalysisInput{ + method: method, + service: service, + isNilResponseAllowed: isNilResponseAllowed, + columns: columns, + } +} + +func (mi *standardMethodAnalysisInput) GetMethod() OperationStore { + return mi.method +} + +func (mi *standardMethodAnalysisInput) GetService() Service { + return mi.service +} + +func (mi *standardMethodAnalysisInput) GetColumns() []ColumnDescriptor { + return mi.columns +} + +func (mi *standardMethodAnalysisInput) IsNilResponseAllowed() bool { + return mi.isNilResponseAllowed +} + +type MethodAnalysisOutput interface { + GetMethod() OperationStore + GetSelectItemsKey() string + GetInsertTabulation() Tabulation + GetSelectTabulation() Tabulation + GetColumns() []ColumnDescriptor + GetItemSchema() (Schema, bool) + GetResponseSchema() (Schema, bool) + IsNilResponseAllowed() bool +} + +type analysisOutput struct { + method OperationStore + selectItemsKey string + insertTabulation Tabulation + selectTabulation Tabulation + columns []ColumnDescriptor + responseSchema Schema + itemSchema Schema + isNilResponseAllowed bool +} + +func (ao *analysisOutput) GetMethod() OperationStore { + return ao.method +} + +func (ao *analysisOutput) IsNilResponseAllowed() bool { + return ao.isNilResponseAllowed +} + +func (ao *analysisOutput) GetSelectItemsKey() string { + return ao.selectItemsKey +} + +func (ao *analysisOutput) GetItemSchema() (Schema, bool) { + return ao.itemSchema, ao.itemSchema != nil +} + +func (ao *analysisOutput) GetResponseSchema() (Schema, bool) { + return ao.responseSchema, ao.responseSchema != nil +} + +func (ao *analysisOutput) GetInsertTabulation() Tabulation { + return ao.insertTabulation +} + +func (ao *analysisOutput) GetSelectTabulation() Tabulation { + return ao.selectTabulation +} + +func (ao *analysisOutput) GetColumns() []ColumnDescriptor { + return ao.columns +} + +func newMethodAnalysisOutput( + method OperationStore, + selectItemsKey string, + insertTabulation Tabulation, + selectTabulation Tabulation, + columns []ColumnDescriptor, + responseSchema Schema, + itemSchema Schema, + isNilResponseAllowed bool, +) MethodAnalysisOutput { + return &analysisOutput{ + method: method, + selectItemsKey: selectItemsKey, + insertTabulation: insertTabulation, + selectTabulation: selectTabulation, + columns: columns, + responseSchema: responseSchema, + itemSchema: itemSchema, + isNilResponseAllowed: isNilResponseAllowed, + } +} + +func NewMethodAnalyzer() MethodAnalyzer { + return &standardMethodAnalyzer{} +} + +type MethodAnalyzer interface { + AnalyzeUnaryAction(MethodAnalysisInput) (MethodAnalysisOutput, error) +} + +type standardMethodAnalyzer struct{} + +func (ma *standardMethodAnalyzer) AnalyzeUnaryAction( + methodAnalysisInput MethodAnalysisInput, +) (MethodAnalysisOutput, error) { + method := methodAnalysisInput.GetMethod() + svc := methodAnalysisInput.GetService() + service, serviceOk := svc.(OpenAPIService) + if !serviceOk { + return nil, fmt.Errorf("AnalyzeUnaryAction(): service is not an OpenAPIService") + } + isNilResponseAllowed := methodAnalysisInput.IsNilResponseAllowed() + cols := methodAnalysisInput.GetColumns() + + selectItemsKey := method.GetSelectItemsKey() + + schema, mediaType, err := method.GetResponseBodySchemaAndMediaType() + insertTabulation := newNilTabulation(service, "", "") + selectTabulation := newNilTabulation(service, "", "") + if err != nil && !isNilResponseAllowed { + return nil, err + } + if err != nil { + schema = newExmptyObjectStandardSchema(service, "", "") + } + itemSchema := schema + if err == nil { + itemObjS, selectItemsKeyRet, err := schema.GetSelectSchema(method.GetSelectItemsKey(), mediaType) + if selectItemsKeyRet != "" { + selectItemsKey = selectItemsKeyRet + } + itemSchema = itemObjS + // rscStr, _ := tbl.GetResourceStr() + unsuitableSchemaMsg := "analyzeUnarySelection(): schema unsuitable for select query" + if err != nil && !isNilResponseAllowed { + return nil, err + } + // rscStr, _ := tbl.GetResourceStr() + if itemObjS == nil && !isNilResponseAllowed { + return nil, fmt.Errorf("%s", unsuitableSchemaMsg) + } + if len(cols) == 0 && itemObjS != nil { + cols = itemObjS.getPropertiesColumns() + // TODO: order + } + insertTabulation = itemObjS.Tabulate(false, "") + + selectTabulation = itemObjS.Tabulate(true, "") + } + + return newMethodAnalysisOutput( + method, + selectItemsKey, + insertTabulation, + selectTabulation, + cols, + schema, + itemSchema, + isNilResponseAllowed, + ), nil +} diff --git a/anysdk/expectedResponse.go b/anysdk/expectedResponse.go index 2d708e3..9c05034 100644 --- a/anysdk/expectedResponse.go +++ b/anysdk/expectedResponse.go @@ -1,5 +1,7 @@ package anysdk +import "github.com/getkin/kin-openapi/openapi3" + var ( _ ExpectedResponse = &standardExpectedResponse{} ) @@ -9,6 +11,8 @@ type ExpectedResponse interface { GetOpenAPIDocKey() string GetObjectKey() string GetSchema() Schema + getOverrideSchema() (*openapi3.Schema, bool) + GetTransform() (Transform, bool) // setSchema(Schema) setBodyMediaType(string) @@ -20,6 +24,8 @@ type standardExpectedResponse struct { OpenAPIDocKey string `json:"openAPIDocKey,omitempty" yaml:"openAPIDocKey,omitempty"` ObjectKey string `json:"objectKey,omitempty" yaml:"objectKey,omitempty"` Schema Schema + OverrideSchema *openapi3.Schema `json:"schema_override,omitempty" yaml:"schema_override,omitempty"` + Transform *standardTransform `json:"transform,omitempty" yaml:"transform,omitempty"` } func (er *standardExpectedResponse) setBodyMediaType(s string) { @@ -43,5 +49,21 @@ func (er *standardExpectedResponse) GetObjectKey() string { } func (er *standardExpectedResponse) GetSchema() Schema { + if er.OverrideSchema != nil { + return newSchema(er.OverrideSchema, nil, "", "") + } return er.Schema } + +func (er *standardExpectedResponse) getOverrideSchema() (*openapi3.Schema, bool) { + isNilSchema := er.OverrideSchema == nil + if isNilSchema { + return nil, false + } + overrideSchema := er.OverrideSchema + return overrideSchema, true +} + +func (er *standardExpectedResponse) GetTransform() (Transform, bool) { + return er.Transform, er.Transform != nil +} diff --git a/anysdk/http.go b/anysdk/http.go index 6088961..f2147b7 100644 --- a/anysdk/http.go +++ b/anysdk/http.go @@ -154,14 +154,15 @@ type HttpParameters interface { GetRemainingQueryParamsFlatMap(keysRemaining map[string]interface{}) (map[string]interface{}, error) GetServerParameterFlatMap() (map[string]interface{}, error) SetResponseBodyParam(key string, val interface{}) - SetServerParam(key string, svc Service, val interface{}) + SetServerParam(key string, svc OpenAPIService, val interface{}) SetRequestBodyParam(key string, val interface{}) SetRequestBody(map[string]interface{}) GetRequestBody() map[string]interface{} + GetInlineParameterFlatMap() (map[string]interface{}, error) } type standardHttpParameters struct { - opStore OperationStore + opStore StandardOperationStore CookieParams ParamMap HeaderParams ParamMap PathParams ParamMap @@ -169,11 +170,12 @@ type standardHttpParameters struct { RequestBody BodyMap ResponseBody BodyMap ServerParams ParamMap + InlineParams ParamMap Unassigned ParamMap Region EncodableString } -func NewHttpParameters(method OperationStore) HttpParameters { +func NewHttpParameters(method StandardOperationStore) HttpParameters { return &standardHttpParameters{ opStore: method, CookieParams: make(ParamMap), @@ -183,6 +185,7 @@ func NewHttpParameters(method OperationStore) HttpParameters { RequestBody: make(BodyMap), ResponseBody: make(BodyMap), ServerParams: make(ParamMap), + InlineParams: make(ParamMap), Unassigned: make(ParamMap), } } @@ -218,7 +221,7 @@ func (hp *standardHttpParameters) SetResponseBodyParam(key string, val interface hp.ResponseBody[key] = val } -func (hp *standardHttpParameters) SetServerParam(key string, svc Service, val interface{}) { +func (hp *standardHttpParameters) SetServerParam(key string, svc OpenAPIService, val interface{}) { hp.ServerParams[key] = NewParameterBinding(NewParameter(&openapi3.Parameter{In: "server"}, svc), val) } @@ -273,6 +276,10 @@ func (hp *standardHttpParameters) StoreParameter(param Addressable, val interfac hp.ServerParams[param.GetName()] = NewParameterBinding(param, val) return } + if param.GetLocation() == "inline" { + hp.InlineParams[param.GetName()] = NewParameterBinding(param, val) + return + } } func (hp *standardHttpParameters) GetParameter(paramName, paramIn string) (ParameterBinding, bool) { @@ -305,7 +312,7 @@ func (hp *standardHttpParameters) GetParameter(paramName, paramIn string) (Param return rv, true } if paramIn == "server" { - rv, ok := hp.CookieParams[paramName] + rv, ok := hp.ServerParams[paramName] if !ok { return nil, false } @@ -325,7 +332,7 @@ func (hp *standardHttpParameters) processFuncHTTPParam(key string, param interfa case *sqlparser.AliasedExpr: switch argExpr := ex.Expr.(type) { case *sqlparser.SQLVal: - queryTransposer := querytranspose.NewQueryTransposer(hp.opStore.GetQueryTransposeAlgorithm(), argExpr.Val, key) + queryTransposer := querytranspose.NewQueryTransposer(hp.opStore.getQueryTransposeAlgorithm(), argExpr.Val, key) return queryTransposer.Transpose() default: return nil, fmt.Errorf("cannot process json function underlying arg of type = '%T'", argExpr) @@ -411,6 +418,18 @@ func (hp *standardHttpParameters) GetServerParameterFlatMap() (map[string]interf return rv, nil } +func (hp *standardHttpParameters) GetInlineParameterFlatMap() (map[string]interface{}, error) { + rv := make(map[string]interface{}) + visited := make(map[string]struct{}) + for k, v := range hp.InlineParams { + err := hp.updateStuff(k, v, rv, visited) + if err != nil { + return nil, err + } + } + return rv, nil +} + func (hp *standardHttpParameters) GetRemainingQueryParamsFlatMap(keysRemaining map[string]interface{}) (map[string]interface{}, error) { rv := make(map[string]interface{}) visited := make(map[string]struct{}) diff --git a/anysdk/http_armoury_params.go b/anysdk/http_armoury_params.go index 64b814a..64bdff5 100644 --- a/anysdk/http_armoury_params.go +++ b/anysdk/http_armoury_params.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" + "github.com/stackql/any-sdk/pkg/client" "github.com/stackql/any-sdk/pkg/internaldto" ) @@ -17,6 +18,7 @@ type HTTPArmouryParameters interface { GetParameters() HttpParameters GetQuery() url.Values GetRequest() *http.Request + GetArgList() client.AnySdkArgList SetBodyBytes(b []byte) SetHeaderKV(k string, v []string) SetNextPage(ops OperationStore, token string, tokenKey internaldto.HTTPElement) (*http.Request, error) @@ -56,6 +58,13 @@ func (hap *standardHTTPArmouryParameters) GetRequest() *http.Request { return hap.request } +func (hap *standardHTTPArmouryParameters) GetArgList() client.AnySdkArgList { + return newAnySdkArgList( + client.HTTP, + newAnySdkHTTPArg(hap.request), + ) +} + func (hap *standardHTTPArmouryParameters) SetRequestBodyMap(body BodyMap) { hap.parameters.SetRequestBody(body) } diff --git a/anysdk/http_preparator_stream.go b/anysdk/http_preparator_stream.go new file mode 100644 index 0000000..ebfeeee --- /dev/null +++ b/anysdk/http_preparator_stream.go @@ -0,0 +1,33 @@ +//nolint:revive,stylecheck // permissable deviation from norm +package anysdk + +var ( + _ HttpPreparatorStream = &httpPreparatorStream{} +) + +type HttpPreparatorStream interface { + Write(HTTPPreparator) error + Next() (HTTPPreparator, bool) +} + +type httpPreparatorStream struct { + sl []HTTPPreparator +} + +func NewHttpPreparatorStream() HttpPreparatorStream { + return &httpPreparatorStream{} +} + +func (s *httpPreparatorStream) Write(p HTTPPreparator) error { + s.sl = append(s.sl, p) + return nil +} + +func (s *httpPreparatorStream) Next() (HTTPPreparator, bool) { + if len(s.sl) < 1 { + return nil, false + } + p := s.sl[0] + s.sl = s.sl[1:] + return p, true +} diff --git a/anysdk/loader.go b/anysdk/loader.go index b410145..5654e55 100644 --- a/anysdk/loader.go +++ b/anysdk/loader.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/fs" - "io/ioutil" "os" "path" "sort" @@ -17,6 +16,7 @@ import ( "github.com/getkin/kin-openapi/openapi3" yamlconv "github.com/ghodss/yaml" "github.com/go-openapi/jsonpointer" + "github.com/stackql/any-sdk/pkg/client" yaml "gopkg.in/yaml.v2" ) @@ -27,7 +27,7 @@ const ( var ( IgnoreEmbedded bool OpenapiFileRoot string - _ Loader = &standardLoader{} + _ anySdkLoader = &standardLoader{} ) func init() { @@ -38,28 +38,82 @@ type DiscoveryDoc interface { iDiscoveryDoc() } -type Loader interface { - LoadFromBytes(bytes []byte) (Service, error) - LoadFromBytesWithProvider(bytes []byte, prov Provider) (Service, error) - LoadFromBytesAndResources(rr ResourceRegister, resourceKey string, bytes []byte) (Service, error) +type anySdkLoader interface { + loadFromBytes(bytes []byte) (OpenAPIService, error) + loadFromBytesWithProvider(bytes []byte, prov Provider) (OpenAPIService, error) + loadFromBytesAndResources(rr ResourceRegister, resourceKey string, bytes []byte) (OpenAPIService, error) // - extractAndMergeQueryTransposeServiceLevel(svc Service) error + extractAndMergeQueryTransposeServiceLevel(svc OpenAPIService) error + mergeLocalResource( + svc Service, + rsc Resource, + // sr *ServiceRef, + ) error } type standardLoader struct { *openapi3.Loader // - visitedExpectedRequest map[Schema]struct{} - visitedExpectedResponse map[Schema]struct{} - visitedOperation map[*openapi3.Operation]struct{} - visitedOperationStore map[OperationStore]struct{} - visitedPathItem map[*openapi3.PathItem]struct{} + visitedExpectedRequest map[Schema]struct{} + visitedExpectedResponse map[Schema]struct{} + visitedOperation map[*openapi3.Operation]struct{} + visitedOpenAPIOperationStore map[StandardOperationStore]struct{} + visitedPathItem map[*openapi3.PathItem]struct{} } func LoadResourcesShallow(ps ProviderService, bt []byte) (ResourceRegister, error) { return loadResourcesShallow(ps, bt) } +func LoadProviderAndServiceFromPaths( + provFilePath string, + svcFilePath string, +) (Service, error) { + pb, err := os.ReadFile(provFilePath) + if err != nil { + return nil, err + } + prov, err := LoadProviderDocFromBytes(pb) + if err != nil { + return nil, err + } + b, err := os.ReadFile(svcFilePath) + if err != nil { + return nil, err + } + protocolType, err := prov.GetProtocolType() + if err != nil { + return nil, err + } + switch protocolType { + case client.HTTP: + l := newLoader() + svc, err := l.loadFromBytesWithProvider(b, prov) + if err != nil { + return nil, err + } + return svc, nil + case client.LocalTemplated: + rv := new(localTemplatedService) + err = yamlconv.Unmarshal(b, rv) + if err != nil { + return nil, err + } + for _, v := range rv.Rsc { + l := newLoader() + rsc := v + mergeErr := l.mergeLocalResource(rv, rsc) + if mergeErr != nil { + return nil, mergeErr + } + } + rv.Provider = prov + return rv, nil + default: + return nil, fmt.Errorf("loader unsupported protocol type '%v'", protocolType) + } +} + func loadResourcesShallow(ps ProviderService, bt []byte) (ResourceRegister, error) { rv := newStandardResourceRegister() err := yaml.Unmarshal(bt, &rv) @@ -76,7 +130,7 @@ func loadResourcesShallow(ps ProviderService, bt []byte) (ResourceRegister, erro return rv, nil } -func (l *standardLoader) LoadFromBytes(bytes []byte) (Service, error) { +func (l *standardLoader) loadFromBytes(bytes []byte) (OpenAPIService, error) { doc, err := l.LoadFromData(bytes) if err != nil { return nil, err @@ -89,8 +143,8 @@ func (l *standardLoader) LoadFromBytes(bytes []byte) (Service, error) { return svc, nil } -func (l *standardLoader) LoadFromBytesWithProvider(bytes []byte, prov Provider) (Service, error) { - svc, err := l.LoadFromBytes(bytes) +func (l *standardLoader) loadFromBytesWithProvider(bytes []byte, prov Provider) (OpenAPIService, error) { + svc, err := l.loadFromBytes(bytes) if err != nil { return nil, err } @@ -98,7 +152,7 @@ func (l *standardLoader) LoadFromBytesWithProvider(bytes []byte, prov Provider) return svc, nil } -func (l *standardLoader) LoadFromBytesAndResources(rr ResourceRegister, resourceKey string, bytes []byte) (Service, error) { +func (l *standardLoader) loadFromBytesAndResources(rr ResourceRegister, resourceKey string, bytes []byte) (OpenAPIService, error) { doc, err := l.LoadFromData(bytes) if err != nil { return nil, err @@ -116,10 +170,10 @@ func (l *standardLoader) LoadFromBytesAndResources(rr ResourceRegister, resource return svc, nil } -func (l *standardLoader) extractResources(svc Service) error { - rscs, ok := svc.GetComponents().Extensions[ExtensionKeyResources] +func (l *standardLoader) extractResources(svc OpenAPIService) error { + rscs, ok := svc.getComponents().Extensions[ExtensionKeyResources] if !ok { - return fmt.Errorf("Service.extractResources() failure") + return fmt.Errorf("OpenAPIService.extractResources() failure") } var bt []byte var err error @@ -144,7 +198,7 @@ func (l *standardLoader) extractResources(svc Service) error { return l.mergeResources(svc, castMap, nil) } -func (l *standardLoader) extractAndMergeGraphQL(operation OperationStore) error { +func (l *standardLoader) extractAndMergeGraphQL(operation StandardOperationStore) error { if operation.GetOperationRef() == nil || operation.GetOperationRef().Value == nil { return nil } @@ -192,7 +246,7 @@ func extractStackQLConfig(qt interface{}) (StackQLConfig, error) { return &rv, nil } -func (l *standardLoader) extractAndMergeQueryTransposeOpLevel(_ OperationStore) error { +func (l *standardLoader) extractAndMergeQueryTransposeOpLevel(_ StandardOperationStore) error { // if operation.GetOperationRef() == nil || operation.GetOperationRef().Value == nil { // return nil // } @@ -208,7 +262,7 @@ func (l *standardLoader) extractAndMergeQueryTransposeOpLevel(_ OperationStore) return nil } -func (l *standardLoader) extractAndMergeQueryTransposeServiceLevel(svc Service) error { +func (l *standardLoader) extractAndMergeQueryTransposeServiceLevel(svc OpenAPIService) error { qt, ok := svc.getExtension(ExtensionKeyConfig) if !ok { return nil @@ -221,7 +275,7 @@ func (l *standardLoader) extractAndMergeQueryTransposeServiceLevel(svc Service) return nil } -func (l *standardLoader) extractAndMergeConfigServiceLevel(svc Service) error { +func (l *standardLoader) extractAndMergeConfigServiceLevel(svc OpenAPIService) error { qt, ok := svc.getExtension(ExtensionKeyConfig) if !ok { return nil @@ -234,7 +288,7 @@ func (l *standardLoader) extractAndMergeConfigServiceLevel(svc Service) error { return nil } -func (l *standardLoader) mergeResources(svc Service, rscMap map[string]Resource, sdRef *ServiceRef) error { +func (l *standardLoader) mergeResources(svc OpenAPIService, rscMap map[string]Resource, sdRef *ServiceRef) error { rscCast := make(map[string]*standardResource, len(rscMap)) for k, rsc := range rscMap { rscCast[k] = rsc.(*standardResource) @@ -254,7 +308,7 @@ func (l *standardLoader) mergeResources(svc Service, rscMap map[string]Resource, return nil } -func (l *standardLoader) mergeResourcesScoped(svc Service, svcUrl string, rr ResourceRegister) error { +func (l *standardLoader) mergeResourcesScoped(svc OpenAPIService, svcUrl string, rr ResourceRegister) error { scopedMap := make(map[string]Resource) for k, rsc := range rr.GetResources() { if rr.ObtainServiceDocUrl(k) == svcUrl { @@ -277,7 +331,7 @@ func (l *standardLoader) mergeResourcesScoped(svc Service, svcUrl string, rr Res return nil } -func (l *standardLoader) mergeResource(svc Service, +func (l *standardLoader) mergeResource(svc OpenAPIService, rsc Resource, sr *ServiceRef, ) error { @@ -308,8 +362,6 @@ func (l *standardLoader) mergeResource(svc Service, if err != nil { return err } - // iv := openapi3.Servers(svc.GetServers()) - // v.setServers(&iv) rsc.setMethod(k, &v) } for sqlVerb, dir := range rsc.getSQLVerbs() { @@ -339,6 +391,71 @@ func (l *standardLoader) mergeResource(svc Service, return propogateErr } +func (l *standardLoader) mergeLocalResource( + svc Service, + rsc Resource, + // sr *ServiceRef, +) error { + // rsc.setService(svc) // must happen before resolving inverses + for k, vOp := range rsc.GetMethods() { + v := vOp + v.setResource(rsc) + rsc.setMethod(k, &v) + } + // v := vOp + // v.setMethodKey(k) + // // TODO: replicate this for the damned inverse + // err := l.resolveOperationRef(svc, rsc, &v, v.GetPathRef(), sr) + // if err != nil { + // return err + // } + // req, reqExists := v.GetRequest() + // if !reqExists && v.GetOperationRef().Value.RequestBody != nil { + // req = &standardExpectedRequest{} + // v.setRequest(req.(*standardExpectedRequest)) + // } + // err = l.resolveExpectedRequest(svc, v.GetOperationRef().Value, req) + // if err != nil { + // return err + // } + // response, responseExists := v.GetResponse() + // if !responseExists && v.GetOperationRef().Value.Responses != nil { + // response = &standardExpectedResponse{} + // v.setResponse(response.(*standardExpectedResponse)) + // } + // err = l.resolveExpectedResponse(svc, v.GetOperationRef().Value, response) + // if err != nil { + // return err + // } + // rsc.setMethod(k, &v) + // } + for sqlVerb, dir := range rsc.getSQLVerbs() { + for i, v := range dir { + cur := v + err := l.resolveSQLVerb(rsc, &cur, sqlVerb) + if err != nil { + return err + } + rsc.mutateSQLVerb(sqlVerb, i, cur) + } + } + // TODO: add second pass for inverse ops + for sqlVerb, dir := range rsc.getSQLVerbs() { + for i, v := range dir { + cur := v + err := l.latePassResolveInverse(svc, &cur) + if err != nil { + return err + } + rsc.mutateSQLVerb(sqlVerb, i, cur) + } + } + // rsc.setProvider(svc.getProvider()) + // rsc.setProviderService(svc.getProviderService()) + propogateErr := rsc.propogateToConfig() + return propogateErr +} + func (svc *standardService) ToJson() ([]byte, error) { return svc.MarshalJSON() } @@ -379,19 +496,44 @@ func (pr *standardProvider) ToYamlFile(filePath string) error { return os.WriteFile(filePath, bytes, ConfigFilesMode) } -func NewLoader() Loader { +func newLoader() anySdkLoader { return &standardLoader{ &openapi3.Loader{Context: context.Background()}, make(map[Schema]struct{}), make(map[Schema]struct{}), make(map[*openapi3.Operation]struct{}), - make(map[OperationStore]struct{}), + make(map[StandardOperationStore]struct{}), make(map[*openapi3.PathItem]struct{}), } } func LoadServiceDocFromBytes(ps ProviderService, bytes []byte) (Service, error) { - return loadServiceDocFromBytes(ps, bytes) + protocolType, err := ps.GetProtocolType() + if err != nil { + return nil, err + } + switch protocolType { + case client.HTTP: + return loadOpenapiServiceDocFromBytes(ps, bytes) + case client.LocalTemplated: + rv := new(localTemplatedService) + err = yamlconv.Unmarshal(bytes, rv) + if err != nil { + return nil, err + } + for _, v := range rv.Rsc { + l := newLoader() + rsc := v + mergeErr := l.mergeLocalResource(rv, rsc) + if mergeErr != nil { + return nil, mergeErr + } + } + rv.ProviderService = ps + return rv, nil + default: + return nil, fmt.Errorf("loader unsupported protocol type '%v'", protocolType) + } } func LoadProviderDocFromBytes(bytes []byte) (Provider, error) { @@ -399,15 +541,15 @@ func LoadProviderDocFromBytes(bytes []byte) (Provider, error) { } func LoadServiceDocFromFile(ps ProviderService, fileName string) (Service, error) { - bytes, err := ioutil.ReadFile(fileName) + bytes, err := os.ReadFile(fileName) if err != nil { return nil, err } - return loadServiceDocFromBytes(ps, bytes) + return LoadServiceDocFromBytes(ps, bytes) } func LoadProviderDocFromFile(fileName string) (Provider, error) { - bytes, err := ioutil.ReadFile(fileName) + bytes, err := os.ReadFile(fileName) if err != nil { return nil, err } @@ -435,6 +577,12 @@ func getServiceDocBytes(url string) ([]byte, error) { return io.ReadAll(f) } +func ReadService(b []byte) (Service, error) { + l := newLoader() + svc, err := l.loadFromBytes(b) + return svc, err +} + func GetResourcesRegisterDocBytes(url string) ([]byte, error) { return getServiceDocBytes(url) } @@ -512,9 +660,9 @@ func getProviderDoc(provider string) (string, error) { return findLatestDoc(path.Join(OpenapiFileRoot, provider)) } -func loadServiceDocFromBytes(ps ProviderService, bytes []byte) (Service, error) { - loader := NewLoader() - rv, err := loader.LoadFromBytes(bytes) +func loadOpenapiServiceDocFromBytes(ps ProviderService, bytes []byte) (OpenAPIService, error) { + loader := newLoader() + rv, err := loader.loadFromBytes(bytes) if err != nil { return nil, err } @@ -531,9 +679,9 @@ func loadServiceDocFromBytes(ps ProviderService, bytes []byte) (Service, error) return rv, nil } -func LoadServiceSubsetDocFromBytes(rr ResourceRegister, resourceKey string, bytes []byte) (Service, error) { - loader := NewLoader() - return loader.LoadFromBytesAndResources(rr, resourceKey, bytes) +func LoadServiceSubsetDocFromBytes(rr ResourceRegister, resourceKey string, bytes []byte) (OpenAPIService, error) { + loader := newLoader() + return loader.loadFromBytesAndResources(rr, resourceKey, bytes) } func loadProviderDocFromBytes(bytes []byte) (Provider, error) { @@ -567,7 +715,7 @@ func resourceregisterLoadBackwardsCompatibility(rr ResourceRegister) { } } -func operationBackwardsCompatibility(component OperationStore, sr *ServiceRef) { +func operationBackwardsCompatibility(component StandardOperationStore, sr *ServiceRef) { // backwards compatibility if component.GetPathRef() != nil { stub := "#/paths/" @@ -581,7 +729,7 @@ func operationBackwardsCompatibility(component OperationStore, sr *ServiceRef) { // } -func (loader *standardLoader) resolveOperationRef(doc Service, rsc Resource, component OperationStore, _ *PathItemRef, sr *ServiceRef) (err error) { +func (loader *standardLoader) resolveOperationRef(doc OpenAPIService, rsc Resource, component StandardOperationStore, _ *PathItemRef, sr *ServiceRef) (err error) { if component == nil { return errors.New("invalid operation: value MUST be an object") @@ -633,7 +781,7 @@ func (loader *standardLoader) resolveOperationRef(doc Service, rsc Resource, com return loader.extractAndMergeGraphQL(component) } -func (loader *standardLoader) resolveContentDefault(content openapi3.Content, svc Service) (Schema, string, bool) { +func (loader *standardLoader) resolveContentDefault(content openapi3.Content, svc OpenAPIService) (Schema, string, bool) { if content == nil { return nil, "", false } @@ -671,7 +819,7 @@ func (loader *standardLoader) findBestResponseDefault(responses openapi3.Respons return nil, false } -func (loader *standardLoader) resolveExpectedRequest(doc Service, op *openapi3.Operation, component ExpectedRequest) (err error) { +func (loader *standardLoader) resolveExpectedRequest(doc OpenAPIService, op *openapi3.Operation, component ExpectedRequest) (err error) { switch component.(type) { case nil: return nil @@ -709,15 +857,15 @@ func (loader *standardLoader) resolveExpectedRequest(doc Service, op *openapi3.O return nil } -func (loader *standardLoader) resolveSQLVerb(rsc Resource, component *OperationStoreRef, sqlVerb string) (err error) { +func (loader *standardLoader) resolveSQLVerb(rsc Resource, component *OpenAPIOperationStoreRef, sqlVerb string) (err error) { if component != nil && component.hasValue() { - if loader.visitedOperationStore == nil { - loader.visitedOperationStore = make(map[OperationStore]struct{}) + if loader.visitedOpenAPIOperationStore == nil { + loader.visitedOpenAPIOperationStore = make(map[StandardOperationStore]struct{}) } - if _, ok := loader.visitedOperationStore[component.Value]; ok { + if _, ok := loader.visitedOpenAPIOperationStore[component.Value]; ok { return nil } - loader.visitedOperationStore[component.Value] = struct{}{} + loader.visitedOpenAPIOperationStore[component.Value] = struct{}{} } resolved, err := resolveSQLVerbFromResource(rsc, component, sqlVerb) @@ -732,7 +880,7 @@ func (loader *standardLoader) resolveSQLVerb(rsc Resource, component *OperationS return nil } -func resolveSQLVerbFromResource(rsc Resource, component *OperationStoreRef, sqlVerb string) (*standardOperationStore, error) { +func resolveSQLVerbFromResource(rsc Resource, component *OpenAPIOperationStoreRef, sqlVerb string) (*standardOpenAPIOperationStore, error) { if component == nil { return nil, fmt.Errorf("operation store ref not supplied") @@ -741,7 +889,7 @@ func resolveSQLVerbFromResource(rsc Resource, component *OperationStoreRef, sqlV if err != nil { return nil, err } - resolved, ok := osv.(*standardOperationStore) + resolved, ok := osv.(*standardOpenAPIOperationStore) if !ok { return nil, fmt.Errorf("operation store ref type '%T' not supported", osv) } @@ -750,7 +898,7 @@ func resolveSQLVerbFromResource(rsc Resource, component *OperationStoreRef, sqlV return rv, nil } -func (l *standardLoader) latePassResolveInverse(svc Service, component *OperationStoreRef) error { +func (l *standardLoader) latePassResolveInverse(svc Service, component *OpenAPIOperationStoreRef) error { if component == nil || component.Value == nil { return fmt.Errorf("late pass: operation store ref not supplied") } @@ -768,7 +916,7 @@ func (l *standardLoader) latePassResolveInverse(svc Service, component *Operatio return nil } -func (loader *standardLoader) resolveExpectedResponse(doc Service, op *openapi3.Operation, component ExpectedResponse) (err error) { +func (loader *standardLoader) resolveExpectedResponse(doc OpenAPIService, op *openapi3.Operation, component ExpectedResponse) (err error) { if component != nil && component.GetSchema() != nil { if loader.visitedExpectedResponse == nil { loader.visitedExpectedResponse = make(map[Schema]struct{}) @@ -784,7 +932,11 @@ func (loader *standardLoader) resolveExpectedResponse(doc Service, op *openapi3. } bmt := component.GetBodyMediaType() ek := component.GetOpenAPIDocKey() - if bmt != "" && ek != "" { + overrideSchema, isOverrideSchema := component.getOverrideSchema() + if isOverrideSchema { + s := newSchema(overrideSchema, doc, "", "") + component.setSchema(s) + } else if bmt != "" && ek != "" { ekObj, ok := op.Responses[ek] if !ok || ekObj.Value == nil || ekObj.Value.Content == nil || ekObj.Value.Content[bmt] == nil || ekObj.Value.Content[bmt].Schema == nil || ekObj.Value.Content[bmt].Schema.Value == nil { return nil diff --git a/anysdk/loader_test.go b/anysdk/loader_test.go index 3ac424d..46b40c3 100644 --- a/anysdk/loader_test.go +++ b/anysdk/loader_test.go @@ -1,4 +1,4 @@ -package anysdk_test +package anysdk import ( "encoding/json" @@ -7,12 +7,24 @@ import ( "path" "testing" - . "github.com/stackql/any-sdk/anysdk" "github.com/stackql/any-sdk/pkg/fileutil" "gotest.tools/assert" ) +var ( + awsTestableVersions = []string{ + "v0.1.0", + } + oktaTestableVersions = []string{ + "v0.1.0", + } + googleTestableVersions = []string{ + // "v0.1.0", + "v0.1.2", + } +) + func setupFileRoot(t *testing.T) { var err error OpenapiFileRoot, err = fileutil.GetFilePathFromRepositoryRoot(path.Join("test", "registry", "src")) @@ -58,9 +70,9 @@ func TestSimpleOktaApplicationServiceReadAndDump(t *testing.T) { t.Fatalf("Test failed: %v", err) } - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) if err != nil { t.Fatalf("Test failed: %v", err) } @@ -83,9 +95,9 @@ func TestSimpleOktaApplicationServiceReadAndDumpString(t *testing.T) { t.Fatalf("Test failed: %v", err) } - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) if err != nil { t.Fatalf("Test failed: %v", err) } @@ -118,9 +130,9 @@ func TestSimpleOktaApplicationServiceJsonReadAndDumpString(t *testing.T) { t.Fatalf("Test failed: %v", err) } - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) if err != nil { t.Fatalf("Test failed: %v", err) } @@ -156,9 +168,9 @@ func TestSimpleAWSec2ServiceJsonReadAndDumpString(t *testing.T) { t.Fatalf("Test failed: %v", err) } - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) if err != nil { t.Fatalf("Test failed: %v", err) } @@ -225,14 +237,14 @@ func TestSimpleGoogleComputeServiceJsonReadAndDumpString(t *testing.T) { t.Fatalf("Test failed: %v", err) } - l := NewLoader() + l := newLoader() rr, err := LoadResourcesShallow(ps, br) if err != nil { t.Fatalf("Test failed: %v", err) } - svc, err := l.LoadFromBytesAndResources(rr, "subnetworks", b) + svc, err := l.loadFromBytesAndResources(rr, "subnetworks", b) if err != nil { t.Fatalf("Test failed: %v", err) } diff --git a/anysdk/metadata.go b/anysdk/metadata.go index a815d09..2fe2798 100644 --- a/anysdk/metadata.go +++ b/anysdk/metadata.go @@ -60,11 +60,11 @@ func GetResourcesHeader(extended bool) []string { } type MetadataStore struct { - Store map[string]Service + Store map[string]OpenAPIService } -func (ms *MetadataStore) GetServices() ([]Service, error) { - var retVal []Service +func (ms *MetadataStore) GetServices() ([]OpenAPIService, error) { + var retVal []OpenAPIService for _, svc := range ms.Store { retVal = append(retVal, svc) } diff --git a/anysdk/methodSet.go b/anysdk/methodSet.go index 818d461..b45b0a1 100644 --- a/anysdk/methodSet.go +++ b/anysdk/methodSet.go @@ -1,25 +1,25 @@ package anysdk -type MethodSet []OperationStore +type MethodSet []StandardOperationStore -func (ms MethodSet) GetFirstMatch(params map[string]interface{}) (OperationStore, map[string]interface{}, bool) { +func (ms MethodSet) GetFirstMatch(params map[string]interface{}) (StandardOperationStore, map[string]interface{}, bool) { return ms.getFirstMatch(params) } -func (ms MethodSet) GetFirst() (OperationStore, string, bool) { +func (ms MethodSet) GetFirst() (StandardOperationStore, string, bool) { return ms.getFirst() } -func (ms MethodSet) getFirstMatch(params map[string]interface{}) (OperationStore, map[string]interface{}, bool) { +func (ms MethodSet) getFirstMatch(params map[string]interface{}) (StandardOperationStore, map[string]interface{}, bool) { for _, m := range ms { - if remainingParams, ok := m.ParameterMatch(params); ok { + if remainingParams, ok := m.parameterMatch(params); ok { return m, remainingParams, true } } return nil, params, false } -func (ms MethodSet) getFirst() (OperationStore, string, bool) { +func (ms MethodSet) getFirst() (StandardOperationStore, string, bool) { for _, m := range ms { return m, m.getName(), true } diff --git a/anysdk/methods.go b/anysdk/methods.go index 8e67b36..a10b5b3 100644 --- a/anysdk/methods.go +++ b/anysdk/methods.go @@ -4,17 +4,17 @@ import ( "fmt" ) -type Methods map[string]standardOperationStore +type Methods map[string]standardOpenAPIOperationStore -func (ms Methods) FindMethod(key string) (OperationStore, error) { +func (ms Methods) FindMethod(key string) (StandardOperationStore, error) { if m, ok := ms[key]; ok { return &m, nil } return nil, fmt.Errorf("could not find method for key = '%s'", key) } -func (ms Methods) OrderMethods() ([]OperationStore, error) { - var selectBin, insertBin, deleteBin, updateBin, replaceBin, execBin []OperationStore +func (ms Methods) OrderMethods() ([]StandardOperationStore, error) { + var selectBin, insertBin, deleteBin, updateBin, replaceBin, execBin []StandardOperationStore for k, pv := range ms { v := pv switch v.GetSQLVerb() { @@ -42,12 +42,12 @@ func (ms Methods) OrderMethods() ([]OperationStore, error) { execBin = append(execBin, &v) } } - sortOperationStoreSlices(selectBin, insertBin, deleteBin, updateBin, replaceBin, execBin) - rv := combineOperationStoreSlices(selectBin, insertBin, deleteBin, updateBin, replaceBin, execBin) + sortOpenAPIOperationStoreSlices(selectBin, insertBin, deleteBin, updateBin, replaceBin, execBin) + rv := combineOpenAPIOperationStoreSlices(selectBin, insertBin, deleteBin, updateBin, replaceBin, execBin) return rv, nil } -func (ms Methods) FindFromSelector(sel OperationSelector) (OperationStore, error) { +func (ms Methods) FindFromSelector(sel OperationSelector) (StandardOperationStore, error) { for _, m := range ms { if m.GetSQLVerb() == sel.GetSQLVerb() { return &m, nil diff --git a/anysdk/mock_http.go b/anysdk/mock_http.go index 17b91c9..b5b4e92 100644 --- a/anysdk/mock_http.go +++ b/anysdk/mock_http.go @@ -59,7 +59,16 @@ func getMockHttpRegistry(vc RegistryConfig) (RegistryAPI, error) { if err != nil { return nil, err } - return NewRegistry(RegistryConfig{RegistryURL: defaultRegistryUrlString, LocalDocRoot: localRegPath, SrcPrefix: vc.SrcPrefix, AllowSrcDownload: vc.AllowSrcDownload}, rt) + return NewRegistry( + RegistryConfig{ + RegistryURL: defaultRegistryUrlString, + LocalDocRoot: localRegPath, + SrcPrefix: vc.SrcPrefix, + AllowSrcDownload: vc.AllowSrcDownload, + VerfifyConfig: vc.VerfifyConfig, + }, + rt, + ) } func getMockFileRegistry(vc RegistryConfig, registryRoot string, useEmbedded bool) (RegistryAPI, error) { diff --git a/anysdk/operation_inverse.go b/anysdk/operation_inverse.go index bd9410c..33021ce 100644 --- a/anysdk/operation_inverse.go +++ b/anysdk/operation_inverse.go @@ -35,14 +35,14 @@ func (oits operationTokens) GetTokenSemantic(key string) (TokenSemantic, bool) { type OperationInverse interface { JSONLookup(token string) (interface{}, error) - GetOperationStore() (OperationStore, bool) + GetOperationStore() (StandardOperationStore, bool) GetTokens() (OperationTokens, bool) GetParamMap(response.Response) (map[string]interface{}, error) } type operationInverse struct { - OpRef *OperationStoreRef `json:"sqlVerb" yaml:"sqlVerb"` - ReverseTokens operationTokens `json:"tokens,omitempty" yaml:"tokens,omitempty"` + OpRef *OpenAPIOperationStoreRef `json:"sqlVerb" yaml:"sqlVerb"` + ReverseTokens operationTokens `json:"tokens,omitempty" yaml:"tokens,omitempty"` } func (oi *operationInverse) JSONLookup(token string) (interface{}, error) { @@ -56,11 +56,11 @@ func (oi *operationInverse) JSONLookup(token string) (interface{}, error) { } } -func (oi *operationInverse) GetOperationStore() (OperationStore, bool) { - return oi.getOperationStore() +func (oi *operationInverse) GetOperationStore() (StandardOperationStore, bool) { + return oi.getOpenAPIOperationStore() } -func (oi *operationInverse) getOperationStore() (OperationStore, bool) { +func (oi *operationInverse) getOpenAPIOperationStore() (StandardOperationStore, bool) { if oi.OpRef != nil && (oi.OpRef.Ref == "" || oi.OpRef.Value == nil) { return nil, false } diff --git a/anysdk/operation_store.go b/anysdk/operation_store.go index ccb64fb..a2372ae 100644 --- a/anysdk/operation_store.go +++ b/anysdk/operation_store.go @@ -35,10 +35,10 @@ const ( ) var ( - _ OperationStore = &standardOperationStore{} + _ StandardOperationStore = &standardOpenAPIOperationStore{} ) -func sortOperationStoreSlices(opSlices ...[]OperationStore) { +func sortOpenAPIOperationStoreSlices(opSlices ...[]StandardOperationStore) { for _, opSlice := range opSlices { sort.SliceStable(opSlice, func(i, j int) bool { return opSlice[i].GetMethodKey() < opSlice[j].GetMethodKey() @@ -46,8 +46,8 @@ func sortOperationStoreSlices(opSlices ...[]OperationStore) { } } -func combineOperationStoreSlices(opSlices ...[]OperationStore) []OperationStore { - var rv []OperationStore +func combineOpenAPIOperationStoreSlices(opSlices ...[]StandardOperationStore) []StandardOperationStore { + var rv []StandardOperationStore for _, sl := range opSlices { rv = append(rv, sl...) } @@ -64,6 +64,7 @@ type OperationStore interface { GetParameters() map[string]Addressable GetPathItem() *openapi3.PathItem GetAPIMethod() string + GetInline() []string GetOperationRef() *OperationRef GetPathRef() *PathItemRef GetRequest() (ExpectedRequest, bool) @@ -72,14 +73,13 @@ type OperationStore interface { GetParameterizedPath() string GetProviderService() ProviderService GetProvider() Provider - GetService() Service + GetService() OpenAPIService GetResource() Resource - ParameterMatch(params map[string]interface{}) (map[string]interface{}, bool) + parameterMatch(params map[string]interface{}) (map[string]interface{}, bool) GetOperationParameter(key string) (Addressable, bool) - GetQueryTransposeAlgorithm() string GetSelectSchemaAndObjectPath() (Schema, string, error) - ProcessResponse(*http.Response) (ProcessedOperationResponse, error) - Parameterize(prov Provider, parentDoc Service, inputParams HttpParameters, requestBody interface{}) (*openapi3filter.RequestValidationInput, error) + ProcessResponse(*http.Response) (ProcessedOperationResponse, error) // to be removed + parameterize(prov Provider, parentDoc Service, inputParams HttpParameters, requestBody interface{}) (*openapi3filter.RequestValidationInput, error) GetSelectItemsKey() string GetResponseBodySchemaAndMediaType() (Schema, string, error) GetRequiredParameters() map[string]Addressable @@ -102,9 +102,14 @@ type OperationStore interface { RevertRequestBodyAttributeRename(string) (string, error) IsRequestBodyAttributeRenamed(string) bool GetRequiredNonBodyParameters() map[string]Addressable + getServiceNameForProvider() string +} + +type StandardOperationStore interface { + OperationStore // + getQueryTransposeAlgorithm() string getRequiredNonBodyParameters() map[string]Addressable - getServiceNameForProvider() string getDefaultRequestBodyBytes() []byte getBaseRequestBodyBytes() []byte getName() string @@ -120,7 +125,7 @@ type OperationStore interface { setProvider(Provider) setProviderService(ProviderService) setResource(Resource) - setService(Service) + setService(OpenAPIService) setOperationRef(*OperationRef) setPathItem(*openapi3.PathItem) renameRequestBodyAttribute(string) (string, error) @@ -134,31 +139,39 @@ type OperationStore interface { // getRequestBodyAttributeLineage(string) (string, error) } -type standardOperationStore struct { +type standardOpenAPIOperationStore struct { MethodKey string `json:"-" yaml:"-"` SQLVerb string `json:"-" yaml:"-"` GraphQL GraphQL `json:"-" yaml:"-"` StackQLConfig *standardStackQLConfig `json:"config,omitempty" yaml:"config,omitempty"` // Optional parameters. - Parameters map[string]interface{} `json:"parameters,omitempty" yaml:"parameters,omitempty"` - PathItem *openapi3.PathItem `json:"-" yaml:"-"` // Required - APIMethod string `json:"apiMethod" yaml:"apiMethod"` // Required - OperationRef *OperationRef `json:"operation" yaml:"operation"` // Required - PathRef *PathItemRef `json:"path" yaml:"path"` // Deprecated - Request *standardExpectedRequest `json:"request" yaml:"request"` - Response *standardExpectedResponse `json:"response" yaml:"response"` - Servers *openapi3.Servers `json:"servers" yaml:"servers"` - Inverse *operationInverse `json:"inverse" yaml:"inverse"` - ServiceName string `json:"serviceName,omitempty" yaml:"serviceName,omitempty"` + Parameters map[string]map[string]interface{} `json:"parameters,omitempty" yaml:"parameters,omitempty"` + PathItem *openapi3.PathItem `json:"-" yaml:"-"` // Required + APIMethod string `json:"apiMethod" yaml:"apiMethod"` // Required + OperationRef *OperationRef `json:"operation" yaml:"operation"` // Required + InlineOp []string `json:"inline" yaml:"inline"` // Deprecated + PathRef *PathItemRef `json:"path" yaml:"path"` // Deprecated + Request *standardExpectedRequest `json:"request" yaml:"request"` + Response *standardExpectedResponse `json:"response" yaml:"response"` + Servers *openapi3.Servers `json:"servers" yaml:"servers"` + Inverse *operationInverse `json:"inverse" yaml:"inverse"` + ServiceName string `json:"serviceName,omitempty" yaml:"serviceName,omitempty"` // private parameterizedPath string `json:"-" yaml:"-"` ProviderService ProviderService `json:"-" yaml:"-"` // upwards traversal Provider Provider `json:"-" yaml:"-"` // upwards traversal - Service Service `json:"-" yaml:"-"` // upwards traversal + OpenAPIService OpenAPIService `json:"-" yaml:"-"` // upwards traversal Resource Resource `json:"-" yaml:"-"` // upwards traversal } -func (op *standardOperationStore) getXMLDeclaration() string { +func (op *standardOpenAPIOperationStore) GetInline() []string { + if op.InlineOp != nil { + return op.InlineOp + } + return []string{} +} + +func (op *standardOpenAPIOperationStore) getXMLDeclaration() string { rv := "" if op.Request != nil { rv = op.Request.XMLDeclaration @@ -169,17 +182,17 @@ func (op *standardOperationStore) getXMLDeclaration() string { return rv } -func (op *standardOperationStore) getServiceNameForProvider() string { +func (op *standardOpenAPIOperationStore) getServiceNameForProvider() string { if op.ServiceName != "" { return op.ServiceName } - if op.Service != nil { - return op.Service.GetName() + if op.OpenAPIService != nil { + return op.OpenAPIService.GetName() } return "" } -func (op *standardOperationStore) getXMLRootAnnotation() string { +func (op *standardOpenAPIOperationStore) getXMLRootAnnotation() string { rv := "" if op.Request != nil { rv = op.Request.XMLRootAnnotation @@ -187,7 +200,7 @@ func (op *standardOperationStore) getXMLRootAnnotation() string { return rv } -func (op *standardOperationStore) getXMLTransform() string { +func (op *standardOpenAPIOperationStore) getXMLTransform() string { rv := "" if op.Request != nil { rv = op.Request.XMLTransform @@ -198,7 +211,7 @@ func (op *standardOperationStore) getXMLTransform() string { return rv } -func (op *standardOperationStore) getRequestBodyStringifiedPaths() (map[string]struct{}, error) { +func (op *standardOpenAPIOperationStore) getRequestBodyStringifiedPaths() (map[string]struct{}, error) { rv := make(map[string]struct{}) requestBodySchema, schemaErr := op.getRequestBodySchema() if schemaErr != nil { @@ -215,60 +228,60 @@ func (op *standardOperationStore) getRequestBodyStringifiedPaths() (map[string]s return rv, nil } -func NewEmptyOperationStore() OperationStore { - return &standardOperationStore{ - Parameters: make(map[string]interface{}), +func NewEmptyOperationStore() StandardOperationStore { + return &standardOpenAPIOperationStore{ + Parameters: make(map[string]map[string]interface{}), } } -func (op *standardOperationStore) getRequestBodyMediaType() string { +func (op *standardOpenAPIOperationStore) getRequestBodyMediaType() string { if op.Request != nil { return op.Request.BodyMediaType } return "" } -func (op *standardOperationStore) getRequestBodyMediaTypeNormalised() string { +func (op *standardOpenAPIOperationStore) getRequestBodyMediaTypeNormalised() string { return media.NormaliseMediaType(op.getRequestBodyMediaType()) } -func (op *standardOperationStore) setPathItem(pi *openapi3.PathItem) { +func (op *standardOpenAPIOperationStore) setPathItem(pi *openapi3.PathItem) { op.PathItem = pi } -func (op *standardOperationStore) setService(svc Service) { - op.Service = svc +func (op *standardOpenAPIOperationStore) setService(svc OpenAPIService) { + op.OpenAPIService = svc } -func (op *standardOperationStore) setOperationRef(opr *OperationRef) { +func (op *standardOpenAPIOperationStore) setOperationRef(opr *OperationRef) { op.OperationRef = opr } -func (op *standardOperationStore) setProvider(pr Provider) { +func (op *standardOpenAPIOperationStore) setProvider(pr Provider) { op.Provider = pr } -func (op *standardOperationStore) setProviderService(ps ProviderService) { +func (op *standardOpenAPIOperationStore) setProviderService(ps ProviderService) { op.ProviderService = ps } -func (op *standardOperationStore) setResource(rs Resource) { +func (op *standardOpenAPIOperationStore) setResource(rs Resource) { op.Resource = rs } -func (op *standardOperationStore) setServers(servers *openapi3.Servers) { +func (op *standardOpenAPIOperationStore) setServers(servers *openapi3.Servers) { op.Servers = servers } -func (op *standardOperationStore) setGraphQL(gql GraphQL) { +func (op *standardOpenAPIOperationStore) setGraphQL(gql GraphQL) { op.GraphQL = gql } -func (op *standardOperationStore) setRequest(req *standardExpectedRequest) { +func (op *standardOpenAPIOperationStore) setRequest(req *standardExpectedRequest) { op.Request = req } -func (op *standardOperationStore) getDefaultRequestBodyBytes() []byte { +func (op *standardOpenAPIOperationStore) getDefaultRequestBodyBytes() []byte { var rv []byte if op.Request != nil && op.Request.Default != "" { rv = []byte(op.Request.Default) @@ -276,7 +289,7 @@ func (op *standardOperationStore) getDefaultRequestBodyBytes() []byte { return rv } -func (op *standardOperationStore) getBaseRequestBodyBytes() []byte { +func (op *standardOpenAPIOperationStore) getBaseRequestBodyBytes() []byte { var rv []byte if op.Request != nil && op.Request.Base != "" { rv = []byte(op.Request.Base) @@ -284,35 +297,35 @@ func (op *standardOperationStore) getBaseRequestBodyBytes() []byte { return rv } -func (op *standardOperationStore) setResponse(resp *standardExpectedResponse) { +func (op *standardOpenAPIOperationStore) setResponse(resp *standardExpectedResponse) { op.Response = resp } -func (op *standardOperationStore) setMethodKey(methodKey string) { +func (op *standardOpenAPIOperationStore) setMethodKey(methodKey string) { op.MethodKey = methodKey } -func (op *standardOperationStore) setSQLVerb(sqlVerb string) { +func (op *standardOpenAPIOperationStore) setSQLVerb(sqlVerb string) { op.SQLVerb = sqlVerb } -func (op *standardOperationStore) GetMethodKey() string { +func (op *standardOpenAPIOperationStore) GetMethodKey() string { return op.MethodKey } -func (op *standardOperationStore) GetSQLVerb() string { +func (op *standardOpenAPIOperationStore) GetSQLVerb() string { return op.SQLVerb } -func (op *standardOperationStore) GetGraphQL() GraphQL { +func (op *standardOpenAPIOperationStore) GetGraphQL() GraphQL { return op.GraphQL } -func (op *standardOperationStore) GetInverse() (OperationInverse, bool) { +func (op *standardOpenAPIOperationStore) GetInverse() (OperationInverse, bool) { return op.Inverse, op.Inverse != nil } -func (op *standardOperationStore) GetStackQLConfig() StackQLConfig { +func (op *standardOpenAPIOperationStore) GetStackQLConfig() StackQLConfig { rv, isPresent := op.getStackQLConfig() if !isPresent { return nil @@ -320,46 +333,46 @@ func (op *standardOperationStore) GetStackQLConfig() StackQLConfig { return rv } -func (op *standardOperationStore) getStackQLConfig() (StackQLConfig, bool) { +func (op *standardOpenAPIOperationStore) getStackQLConfig() (StackQLConfig, bool) { rv := op.StackQLConfig return rv, rv != nil } -func (op *standardOperationStore) GetAPIMethod() string { +func (op *standardOpenAPIOperationStore) GetAPIMethod() string { return op.APIMethod } -func (op *standardOperationStore) GetOperationRef() *OperationRef { +func (op *standardOpenAPIOperationStore) GetOperationRef() *OperationRef { return op.OperationRef } -func (op *standardOperationStore) GetPathRef() *PathItemRef { +func (op *standardOpenAPIOperationStore) GetPathRef() *PathItemRef { return op.PathRef } -func (op *standardOperationStore) GetPathItem() *openapi3.PathItem { +func (op *standardOpenAPIOperationStore) GetPathItem() *openapi3.PathItem { return op.PathItem } -func (op *standardOperationStore) GetRequest() (ExpectedRequest, bool) { +func (op *standardOpenAPIOperationStore) GetRequest() (ExpectedRequest, bool) { if op.Request == nil { return nil, false } return op.Request, true } -func (op *standardOperationStore) GetResponse() (ExpectedResponse, bool) { +func (op *standardOpenAPIOperationStore) GetResponse() (ExpectedResponse, bool) { if op.Response == nil { return nil, false } return op.Response, true } -func (op *standardOperationStore) GetServers() (openapi3.Servers, bool) { +func (op *standardOpenAPIOperationStore) GetServers() (openapi3.Servers, bool) { return op.getServers() } -func (op *standardOperationStore) getServers() (openapi3.Servers, bool) { +func (op *standardOpenAPIOperationStore) getServers() (openapi3.Servers, bool) { servers := getServersFromHeirarchy(op) if len(servers) > 0 { return servers, true @@ -367,62 +380,58 @@ func (op *standardOperationStore) getServers() (openapi3.Servers, bool) { if op.Servers != nil { return *(op.Servers), true } - if op.Service != nil { - return op.Service.GetServers() + if op.OpenAPIService != nil { + return op.OpenAPIService.GetServers() } return nil, false } -func (op *standardOperationStore) GetProviderService() ProviderService { +func (op *standardOpenAPIOperationStore) GetProviderService() ProviderService { return op.ProviderService } -func (op *standardOperationStore) GetProvider() Provider { +func (op *standardOpenAPIOperationStore) GetProvider() Provider { return op.Provider } -func (op *standardOperationStore) GetService() Service { - return op.Service +func (op *standardOpenAPIOperationStore) GetService() OpenAPIService { + return op.OpenAPIService } -func (op *standardOperationStore) GetResource() Resource { +func (op *standardOpenAPIOperationStore) GetResource() Resource { return op.Resource } -func (op *standardOperationStore) ParameterMatch(params map[string]interface{}) (map[string]interface{}, bool) { - return op.parameterMatch(params) -} - -func (op *standardOperationStore) GetViewsForSqlDialect(sqlDialect string) ([]View, bool) { +func (op *standardOpenAPIOperationStore) GetViewsForSqlDialect(sqlDialect string) ([]View, bool) { if op.StackQLConfig != nil { return op.StackQLConfig.GetViewsForSqlDialect(sqlDialect, "") } return []View{}, false } -func (op *standardOperationStore) GetQueryTransposeAlgorithm() string { +func (op *standardOpenAPIOperationStore) getQueryTransposeAlgorithm() string { if op.StackQLConfig != nil { transpose, transposeExists := op.StackQLConfig.GetQueryTranspose() if transposeExists && transpose.GetAlgorithm() != "" { return transpose.GetAlgorithm() } } - if op.Resource != nil && op.Resource.GetQueryTransposeAlgorithm() != "" { - return op.Resource.GetQueryTransposeAlgorithm() + if op.Resource != nil && op.Resource.getQueryTransposeAlgorithm() != "" { + return op.Resource.getQueryTransposeAlgorithm() } - if op.Service != nil && op.Service.GetQueryTransposeAlgorithm() != "" { - return op.Service.GetQueryTransposeAlgorithm() + if op.OpenAPIService != nil && op.OpenAPIService.getQueryTransposeAlgorithm() != "" { + return op.OpenAPIService.getQueryTransposeAlgorithm() } - if op.ProviderService != nil && op.ProviderService.GetQueryTransposeAlgorithm() != "" { - return op.ProviderService.GetQueryTransposeAlgorithm() + if op.ProviderService != nil && op.ProviderService.getQueryTransposeAlgorithm() != "" { + return op.ProviderService.getQueryTransposeAlgorithm() } - if op.Provider != nil && op.Provider.GetQueryTransposeAlgorithm() != "" { - return op.Provider.GetQueryTransposeAlgorithm() + if op.Provider != nil && op.Provider.getQueryTransposeAlgorithm() != "" { + return op.Provider.getQueryTransposeAlgorithm() } return "" } -func (op *standardOperationStore) GetRequestTranslateAlgorithm() string { +func (op *standardOpenAPIOperationStore) GetRequestTranslateAlgorithm() string { if op.StackQLConfig != nil { translate, translateExists := op.StackQLConfig.GetRequestTranslate() if translateExists && translate.GetAlgorithm() != "" { @@ -432,8 +441,8 @@ func (op *standardOperationStore) GetRequestTranslateAlgorithm() string { if op.Resource != nil && op.Resource.GetRequestTranslateAlgorithm() != "" { return op.Resource.GetRequestTranslateAlgorithm() } - if op.Service != nil && op.Service.GetRequestTranslateAlgorithm() != "" { - return op.Service.GetRequestTranslateAlgorithm() + if op.OpenAPIService != nil && op.OpenAPIService.getRequestTranslateAlgorithm() != "" { + return op.OpenAPIService.getRequestTranslateAlgorithm() } if op.ProviderService != nil && op.ProviderService.GetRequestTranslateAlgorithm() != "" { return op.ProviderService.GetRequestTranslateAlgorithm() @@ -444,7 +453,7 @@ func (op *standardOperationStore) GetRequestTranslateAlgorithm() string { return "" } -func (op *standardOperationStore) GetPaginationRequestTokenSemantic() (TokenSemantic, bool) { +func (op *standardOpenAPIOperationStore) GetPaginationRequestTokenSemantic() (TokenSemantic, bool) { if op.StackQLConfig != nil { pag, pagExists := op.StackQLConfig.GetPagination() if pagExists && pag.GetRequestToken() != nil { @@ -456,8 +465,8 @@ func (op *standardOperationStore) GetPaginationRequestTokenSemantic() (TokenSema return ts, true } } - if op.Service != nil { - if ts, ok := op.Service.GetPaginationRequestTokenSemantic(); ok { + if op.OpenAPIService != nil { + if ts, ok := op.OpenAPIService.getPaginationRequestTokenSemantic(); ok { return ts, true } } @@ -474,7 +483,7 @@ func (op *standardOperationStore) GetPaginationRequestTokenSemantic() (TokenSema return nil, false } -func (op *standardOperationStore) GetPaginationResponseTokenSemantic() (TokenSemantic, bool) { +func (op *standardOpenAPIOperationStore) GetPaginationResponseTokenSemantic() (TokenSemantic, bool) { if op.StackQLConfig != nil { pag, pagExists := op.StackQLConfig.GetPagination() if pagExists && pag.GetResponseToken() != nil { @@ -486,25 +495,25 @@ func (op *standardOperationStore) GetPaginationResponseTokenSemantic() (TokenSem return ts, true } } - if op.Service != nil { - if ts, ok := op.Service.GetPaginationResponseTokenSemantic(); ok { + if op.OpenAPIService != nil { + if ts, ok := op.OpenAPIService.getPaginationResponseTokenSemantic(); ok { return ts, true } } if op.ProviderService != nil { - if ts, ok := op.ProviderService.GetPaginationResponseTokenSemantic(); ok { + if ts, ok := op.ProviderService.getPaginationResponseTokenSemantic(); ok { return ts, true } } if op.Provider != nil { - if ts, ok := op.ProviderService.GetPaginationResponseTokenSemantic(); ok { + if ts, ok := op.ProviderService.getPaginationResponseTokenSemantic(); ok { return ts, true } } return nil, false } -func (op *standardOperationStore) parameterMatch(params map[string]interface{}) (map[string]interface{}, bool) { +func (op *standardOpenAPIOperationStore) parameterMatch(params map[string]interface{}) (map[string]interface{}, bool) { copiedParams := make(map[string]interface{}) for k, v := range params { copiedParams[k] = v @@ -545,27 +554,27 @@ func (op *standardOperationStore) parameterMatch(params map[string]interface{}) return copiedParams, false } -func (op *standardOperationStore) GetParameterizedPath() string { +func (op *standardOpenAPIOperationStore) GetParameterizedPath() string { return op.parameterizedPath } -func (op *standardOperationStore) GetOptimalResponseMediaType() string { +func (op *standardOpenAPIOperationStore) GetOptimalResponseMediaType() string { return op.getOptimalResponseMediaType() } -func (op *standardOperationStore) getOptimalResponseMediaType() string { +func (op *standardOpenAPIOperationStore) getOptimalResponseMediaType() string { if op.Response != nil && op.Response.BodyMediaType != "" { return op.Response.BodyMediaType } return media.MediaTypeJson } -func (op *standardOperationStore) IsNullary() bool { +func (op *standardOpenAPIOperationStore) IsNullary() bool { rbs, _, _ := op.GetResponseBodySchemaAndMediaType() return rbs == nil } -func (m *standardOperationStore) KeyExists(lhs string) bool { +func (m *standardOpenAPIOperationStore) KeyExists(lhs string) bool { if lhs == MethodName { return true } @@ -600,26 +609,26 @@ func (m *standardOperationStore) KeyExists(lhs string) bool { return false } -func (m *standardOperationStore) GetSelectItemsKey() string { +func (m *standardOpenAPIOperationStore) GetSelectItemsKey() string { return m.getSelectItemsKeySimple() } -func (m *standardOperationStore) GetUnionRequiredParameters() (map[string]Addressable, error) { +func (m *standardOpenAPIOperationStore) GetUnionRequiredParameters() (map[string]Addressable, error) { return m.getUnionRequiredParameters() } -func (m *standardOperationStore) getUnionRequiredParameters() (map[string]Addressable, error) { +func (m *standardOpenAPIOperationStore) getUnionRequiredParameters() (map[string]Addressable, error) { return m.Resource.getUnionRequiredParameters(m) } -func (m *standardOperationStore) getSelectItemsKeySimple() string { +func (m *standardOpenAPIOperationStore) getSelectItemsKeySimple() string { if m.Response != nil { return m.Response.ObjectKey } return "" } -func (m *standardOperationStore) GetKey(lhs string) (interface{}, error) { +func (m *standardOpenAPIOperationStore) GetKey(lhs string) (interface{}, error) { val, ok := m.ToPresentationMap(true)[lhs] if !ok { return nil, fmt.Errorf("key '%s' no preset in metadata_method", lhs) @@ -627,7 +636,7 @@ func (m *standardOperationStore) GetKey(lhs string) (interface{}, error) { return val, nil } -func (m *standardOperationStore) GetColumnOrder(extended bool) []string { +func (m *standardOpenAPIOperationStore) GetColumnOrder(extended bool) []string { retVal := []string{ MethodName, RequiredParams, @@ -639,7 +648,7 @@ func (m *standardOperationStore) GetColumnOrder(extended bool) []string { return retVal } -func (m *standardOperationStore) IsAwaitable() bool { +func (m *standardOpenAPIOperationStore) IsAwaitable() bool { rs, _, err := m.GetResponseBodySchemaAndMediaType() if err != nil { return false @@ -647,11 +656,11 @@ func (m *standardOperationStore) IsAwaitable() bool { return strings.HasSuffix(rs.getKey(), "Operation") } -func (m *standardOperationStore) FilterBy(predicate func(interface{}) (ITable, error)) (ITable, error) { +func (m *standardOpenAPIOperationStore) FilterBy(predicate func(interface{}) (ITable, error)) (ITable, error) { return predicate(m) } -func (m *standardOperationStore) GetKeyAsSqlVal(lhs string) (sqltypes.Value, error) { +func (m *standardOpenAPIOperationStore) GetKeyAsSqlVal(lhs string) (sqltypes.Value, error) { val, ok := m.ToPresentationMap(true)[lhs] rv, err := InterfaceToSQLType(val) if !ok { @@ -661,11 +670,11 @@ func (m *standardOperationStore) GetKeyAsSqlVal(lhs string) (sqltypes.Value, err } // This method needs to incorporate request body parameters -func (m *standardOperationStore) GetRequiredParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) GetRequiredParameters() map[string]Addressable { return m.getRequiredParameters() } -func (m *standardOperationStore) getRequestBodyAttributes() (map[string]Addressable, error) { +func (m *standardOpenAPIOperationStore) getRequestBodyAttributes() (map[string]Addressable, error) { s, err := m.getRequestBodySchema() if err != nil { return nil, err @@ -689,7 +698,7 @@ func (m *standardOperationStore) getRequestBodyAttributes() (map[string]Addressa return rv, nil } -func (m *standardOperationStore) getRequestBodyAttributesNoRename() (map[string]Addressable, error) { +func (m *standardOpenAPIOperationStore) getRequestBodyAttributesNoRename() (map[string]Addressable, error) { s, err := m.getRequestBodySchema() if err != nil { return nil, err @@ -709,15 +718,15 @@ func (m *standardOperationStore) getRequestBodyAttributesNoRename() (map[string] return rv, nil } -func (m *standardOperationStore) getRequiredRequestBodyAttributes() (map[string]Addressable, error) { +func (m *standardOpenAPIOperationStore) getRequiredRequestBodyAttributes() (map[string]Addressable, error) { return m.getIndicatedRequestBodyAttributes(true) } -func (m *standardOperationStore) getOptionalRequestBodyAttributes() (map[string]Addressable, error) { +func (m *standardOpenAPIOperationStore) getOptionalRequestBodyAttributes() (map[string]Addressable, error) { return m.getIndicatedRequestBodyAttributes(false) } -func (m *standardOperationStore) getIndicatedRequestBodyAttributes(required bool) (map[string]Addressable, error) { +func (m *standardOpenAPIOperationStore) getIndicatedRequestBodyAttributes(required bool) (map[string]Addressable, error) { rv := make(map[string]Addressable) allAttr, err := m.getRequestBodyAttributes() if err != nil { @@ -731,11 +740,11 @@ func (m *standardOperationStore) getIndicatedRequestBodyAttributes(required bool return rv, nil } -func (m *standardOperationStore) RenameRequestBodyAttribute(k string) (string, error) { +func (m *standardOpenAPIOperationStore) RenameRequestBodyAttribute(k string) (string, error) { return m.renameRequestBodyAttribute(k) } -func (m *standardOperationStore) renameRequestBodyAttribute(k string) (string, error) { +func (m *standardOpenAPIOperationStore) renameRequestBodyAttribute(k string) (string, error) { paramTranslator, translatorInferErr := m.inferTranslator(m.getRequestBodyTranslateAlgorithmString()) if translatorInferErr != nil { return "", translatorInferErr @@ -744,11 +753,11 @@ func (m *standardOperationStore) renameRequestBodyAttribute(k string) (string, e return output, outputErr } -func (m *standardOperationStore) RevertRequestBodyAttributeRename(k string) (string, error) { +func (m *standardOpenAPIOperationStore) RevertRequestBodyAttributeRename(k string) (string, error) { return m.revertRequestBodyAttributeRename(k) } -func (m *standardOperationStore) revertRequestBodyAttributeRename(k string) (string, error) { +func (m *standardOpenAPIOperationStore) revertRequestBodyAttributeRename(k string) (string, error) { paramTranslator, translatorInferErr := m.inferTranslator(m.getRequestBodyTranslateAlgorithmString()) if translatorInferErr != nil { return "", translatorInferErr @@ -757,7 +766,7 @@ func (m *standardOperationStore) revertRequestBodyAttributeRename(k string) (str return output, outputErr } -func (m *standardOperationStore) getRequestBodyAttributeParentKey(algorithm string) (string, bool) { +func (m *standardOpenAPIOperationStore) getRequestBodyAttributeParentKey(algorithm string) (string, bool) { algorithmPrefix := extractAlgorithmPrefix(algorithm) algorithmSuffix := extractAlgorithmSuffix(algorithm, algorithmPrefix) if algorithmPrefix == translateAlgorithmNaiveNaming { @@ -766,15 +775,15 @@ func (m *standardOperationStore) getRequestBodyAttributeParentKey(algorithm stri return "", false } -// func (op *standardOperationStore) getRequestBodyAttributeLineage(rawKey string) (string, error) { +// func (op *standardOpenAPIOperationStore) getRequestBodyAttributeLineage(rawKey string) (string, error) { // return "", nil // } -func (m *standardOperationStore) getDefaultRequestBodyMatcher() fuzzymatch.FuzzyMatcher[string] { +func (m *standardOpenAPIOperationStore) getDefaultRequestBodyMatcher() fuzzymatch.FuzzyMatcher[string] { return requestBodyBaseKeyFuzzyMatcher } -func (m *standardOperationStore) getRequestBodySchemaAttributeMatcher(path string) (fuzzymatch.FuzzyMatcher[string], error) { +func (m *standardOpenAPIOperationStore) getRequestBodySchemaAttributeMatcher(path string) (fuzzymatch.FuzzyMatcher[string], error) { schemaOfInterest, err := m.getRequestBodySchema() if err != nil { return nil, err @@ -822,7 +831,7 @@ func extractAlgorithmPrefix(algorithm string) string { return algorithm } -func (m *standardOperationStore) inferTranslator(algorithm string) (parametertranslate.ParameterTranslator, error) { +func (m *standardOpenAPIOperationStore) inferTranslator(algorithm string) (parametertranslate.ParameterTranslator, error) { algorithmPrefix := extractAlgorithmPrefix(algorithm) algorithmSuffix := extractAlgorithmSuffix(algorithm, algorithmPrefix) switch algorithmPrefix { @@ -847,7 +856,7 @@ func (m *standardOperationStore) inferTranslator(algorithm string) (parametertra } } -func (m *standardOperationStore) getRequestBodyTranslateAlgorithmString() string { +func (m *standardOpenAPIOperationStore) getRequestBodyTranslateAlgorithmString() string { retVal := "" cfg, cfgExists := m.getStackQLConfig() if cfgExists { @@ -862,7 +871,7 @@ func (m *standardOperationStore) getRequestBodyTranslateAlgorithmString() string return retVal } -func (m *standardOperationStore) IsRequestBodyAttributeRenamed(k string) bool { +func (m *standardOpenAPIOperationStore) IsRequestBodyAttributeRenamed(k string) bool { paramTranslator, translatorInferErr := m.inferTranslator(m.getRequestBodyTranslateAlgorithmString()) if translatorInferErr != nil { return false @@ -871,34 +880,48 @@ func (m *standardOperationStore) IsRequestBodyAttributeRenamed(k string) bool { return outputErr == nil } -func (m *standardOperationStore) GetRequiredNonBodyParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) GetRequiredNonBodyParameters() map[string]Addressable { return m.getRequiredNonBodyParameters() } -func (m *standardOperationStore) getRequiredNonBodyParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) getRequiredNonBodyParameters() map[string]Addressable { retVal := make(map[string]Addressable) + for k, v := range m.Parameters { + b, err := json.Marshal(v) + if err != nil { + continue + } + var param openapi3.Parameter + err = json.Unmarshal(b, ¶m) + if err != nil { + continue + } + paramObj := NewParameter(¶m, m.OpenAPIService) + if paramObj.IsRequired() { + retVal[k] = paramObj + } + } if m.PathItem != nil { for _, p := range m.PathItem.Parameters { param := p.Value if param != nil && isOpenapi3ParamRequired(param) { - retVal[param.Name] = NewParameter(p.Value, m.Service) + retVal[param.Name] = NewParameter(p.Value, m.OpenAPIService) } } } - if m.OperationRef == nil || m.OperationRef.Value.Parameters == nil { - + if m.OperationRef == nil || m.OperationRef.Value == nil || m.OperationRef.Value.Parameters == nil { return retVal } for _, p := range m.OperationRef.Value.Parameters { param := p.Value if param != nil && isOpenapi3ParamRequired(param) { - retVal[param.Name] = NewParameter(p.Value, m.Service) + retVal[param.Name] = NewParameter(p.Value, m.OpenAPIService) } } return retVal } -func (m *standardOperationStore) getRequiredParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) getRequiredParameters() map[string]Addressable { retVal := m.getRequiredNonBodyParameters() ss, err := m.getRequiredRequestBodyAttributes() if err != nil { @@ -910,7 +933,7 @@ func (m *standardOperationStore) getRequiredParameters() map[string]Addressable availableServers, availableServersDoExist := m.getServers() if availableServersDoExist { sv := availableServers[0] - serverVarMap := getServerVariablesMap(sv, m.Service) + serverVarMap := getServerVariablesMap(sv, m.OpenAPIService) for k, v := range serverVarMap { retVal[k] = v } @@ -919,20 +942,35 @@ func (m *standardOperationStore) getRequiredParameters() map[string]Addressable } // This method needs to incorporate request body parameters -func (m *standardOperationStore) GetOptionalParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) GetOptionalParameters() map[string]Addressable { return m.getOptionalParameters() } -func (m *standardOperationStore) getOptionalParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) getOptionalParameters() map[string]Addressable { retVal := make(map[string]Addressable) - if m.OperationRef == nil || m.OperationRef.Value.Parameters == nil { + for k, v := range m.Parameters { + b, err := json.Marshal(v) + if err != nil { + continue + } + var param openapi3.Parameter + err = json.Unmarshal(b, ¶m) + if err != nil { + continue + } + paramObj := NewParameter(¶m, m.OpenAPIService) + if !paramObj.IsRequired() { + retVal[k] = paramObj + } + } + if m.OperationRef == nil || m.OperationRef.Value == nil || m.OperationRef.Value.Parameters == nil { return retVal } for _, p := range m.OperationRef.Value.Parameters { param := p.Value // TODO: handle the `?param` where value is not only not required but should NEVER be sent if param != nil && !param.Required { - retVal[param.Name] = NewParameter(p.Value, m.Service) + retVal[param.Name] = NewParameter(p.Value, m.OpenAPIService) } } ss, err := m.getOptionalRequestBodyAttributes() @@ -945,20 +983,20 @@ func (m *standardOperationStore) getOptionalParameters() map[string]Addressable return retVal } -func (ops *standardOperationStore) getMethod() (*openapi3.Operation, error) { +func (ops *standardOpenAPIOperationStore) getMethod() (*openapi3.Operation, error) { if ops.OperationRef != nil && ops.OperationRef.Value != nil { return ops.OperationRef.Value, nil } return nil, fmt.Errorf("no method attached to operation store") } -func (m *standardOperationStore) getNonBodyParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) getNonBodyParameters() map[string]Addressable { retVal := make(map[string]Addressable) if m.PathItem != nil { for _, p := range m.PathItem.Parameters { param := p.Value if param != nil { - retVal[param.Name] = NewParameter(p.Value, m.Service) + retVal[param.Name] = NewParameter(p.Value, m.OpenAPIService) } } } @@ -968,13 +1006,13 @@ func (m *standardOperationStore) getNonBodyParameters() map[string]Addressable { for _, p := range m.OperationRef.Value.Parameters { param := p.Value if param != nil { - retVal[param.Name] = NewParameter(p.Value, m.Service) + retVal[param.Name] = NewParameter(p.Value, m.OpenAPIService) } } return retVal } -func (m *standardOperationStore) GetParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) GetParameters() map[string]Addressable { retVal := m.getNonBodyParameters() ss, err := m.getRequestBodyAttributes() if err != nil { @@ -986,28 +1024,28 @@ func (m *standardOperationStore) GetParameters() map[string]Addressable { return retVal } -func (m *standardOperationStore) GetNonBodyParameters() map[string]Addressable { +func (m *standardOpenAPIOperationStore) GetNonBodyParameters() map[string]Addressable { return m.getNonBodyParameters() } -func (m *standardOperationStore) GetParameter(paramKey string) (Addressable, bool) { +func (m *standardOpenAPIOperationStore) GetParameter(paramKey string) (Addressable, bool) { params := m.GetParameters() rv, ok := params[paramKey] return rv, ok } -func (m *standardOperationStore) GetName() string { +func (m *standardOpenAPIOperationStore) GetName() string { return m.getName() } -func (m *standardOperationStore) getName() string { +func (m *standardOpenAPIOperationStore) getName() string { if m.OperationRef != nil && m.OperationRef.Value != nil && m.OperationRef.Value.OperationID != "" { return m.OperationRef.Value.OperationID } return m.MethodKey } -func (m *standardOperationStore) ToPresentationMap(extended bool) map[string]interface{} { +func (m *standardOpenAPIOperationStore) ToPresentationMap(extended bool) map[string]interface{} { requiredParams := m.getRequiredNonBodyParameters() var requiredParamNames []string for s := range requiredParams { @@ -1034,7 +1072,7 @@ func (m *standardOperationStore) ToPresentationMap(extended bool) map[string]int availableServers, availableServersDoExist := m.getServers() if availableServersDoExist { sv := availableServers[0] - serverVarMap := getServerVariablesMap(sv, m.Service) + serverVarMap := getServerVariablesMap(sv, m.OpenAPIService) for k := range serverVarMap { requiredServerParamNames = append(requiredServerParamNames, k) } @@ -1062,11 +1100,31 @@ func (m *standardOperationStore) ToPresentationMap(extended bool) map[string]int return retVal } -func (op *standardOperationStore) GetOperationParameters() Params { - return NewParameters(op.OperationRef.Value.Parameters, op.Service) +func (op *standardOpenAPIOperationStore) GetOperationParameters() Params { + return NewParameters(op.OperationRef.Value.Parameters, op.OpenAPIService) } -func (op *standardOperationStore) GetOperationParameter(key string) (Addressable, bool) { +func (op *standardOpenAPIOperationStore) GetOperationParameter(key string) (Addressable, bool) { + paramLocal, isParamLocal := op.Parameters[key] + if isParamLocal { + b, err := json.Marshal(paramLocal) + if err != nil { + return nil, false + } + var param openapi3.Parameter + err = json.Unmarshal(b, ¶m) + if err != nil { + return nil, false + } + if param.Name == "" && param.In == "inline" { + param.Name = key + } + paramObj := NewParameter(¶m, op.OpenAPIService) + return paramObj, true + } + if op.OperationRef == nil || op.OperationRef.Value == nil || op.OperationRef.Value.Parameters == nil { + return nil, false + } params := NewParameters(op.OperationRef.Value.Parameters, op.GetService()) if op.OperationRef.Value.Parameters == nil { return nil, false @@ -1074,7 +1132,7 @@ func (op *standardOperationStore) GetOperationParameter(key string) (Addressable return params.GetParameter(key) } -func (op *standardOperationStore) getServerVariable(key string) (*openapi3.ServerVariable, bool) { +func (op *standardOpenAPIOperationStore) getServerVariable(key string) (*openapi3.ServerVariable, bool) { srvs, _ := op.getServers() for _, srv := range srvs { v, ok := srv.Variables[key] @@ -1085,8 +1143,8 @@ func (op *standardOperationStore) getServerVariable(key string) (*openapi3.Serve return nil, false } -func getServersFromHeirarchy(op *standardOperationStore) openapi3.Servers { - if op.OperationRef.Value.Servers != nil && len(*op.OperationRef.Value.Servers) > 0 { +func getServersFromHeirarchy(op *standardOpenAPIOperationStore) openapi3.Servers { + if op.OperationRef != nil && op.OperationRef.Value != nil && op.OperationRef.Value.Servers != nil && len(*op.OperationRef.Value.Servers) > 0 { return *op.OperationRef.Value.Servers } if op.PathItem != nil && len(op.PathItem.Servers) > 0 { @@ -1110,13 +1168,13 @@ func selectServer(servers openapi3.Servers, inputParams map[string]interface{}) return urltranslate.SanitiseServerURL(srvs[0]) } -func (op *standardOperationStore) acceptPathParam(mutableParamMap map[string]interface{}) {} +func (op *standardOpenAPIOperationStore) acceptPathParam(mutableParamMap map[string]interface{}) {} -func (op *standardOperationStore) MarshalBody(body interface{}, expectedRequest ExpectedRequest) ([]byte, error) { +func (op *standardOpenAPIOperationStore) MarshalBody(body interface{}, expectedRequest ExpectedRequest) ([]byte, error) { return op.marshalBody(body, expectedRequest) } -func (op *standardOperationStore) marshalBody(body interface{}, expectedRequest ExpectedRequest) ([]byte, error) { +func (op *standardOpenAPIOperationStore) marshalBody(body interface{}, expectedRequest ExpectedRequest) ([]byte, error) { mediaType := expectedRequest.GetBodyMediaType() if expectedRequest.GetSchema() != nil { mediaType = expectedRequest.GetSchema().extractMediaTypeSynonym(mediaType) @@ -1136,7 +1194,8 @@ func (op *standardOperationStore) marshalBody(body interface{}, expectedRequest return nil, fmt.Errorf("media type = '%s' not supported", expectedRequest.GetBodyMediaType()) } -func (op *standardOperationStore) Parameterize(prov Provider, parentDoc Service, inputParams HttpParameters, requestBody interface{}) (*openapi3filter.RequestValidationInput, error) { +func (op *standardOpenAPIOperationStore) parameterize(prov Provider, parentDoc Service, inputParams HttpParameters, requestBody interface{}) (*openapi3filter.RequestValidationInput, error) { + params := op.OperationRef.Value.Parameters copyParams := make(map[string]interface{}) flatParameters, err := inputParams.ToFlatMap() @@ -1163,7 +1222,7 @@ func (op *standardOperationStore) Parameterize(prov Provider, parentDoc Service, } else if p.Value != nil && p.Value.Schema != nil && p.Value.Schema.Value != nil && p.Value.Schema.Value.Default != nil { prefilledHeader.Set(name, fmt.Sprintf("%v", p.Value.Schema.Value.Default)) } else if isOpenapi3ParamRequired(p.Value) { - return nil, fmt.Errorf("standardOperationStore.Parameterize() failure; missing required header '%s'", name) + return nil, fmt.Errorf("standardOpenAPIOperationStore.parameterize() failure; missing required header '%s'", name) } } if p.Value.In == openapi3.ParameterInPath { @@ -1173,7 +1232,7 @@ func (op *standardOperationStore) Parameterize(prov Provider, parentDoc Service, delete(copyParams, name) } if !present && isOpenapi3ParamRequired(p.Value) { - return nil, fmt.Errorf("standardOperationStore.Parameterize() failure; missing required path parameter '%s'", name) + return nil, fmt.Errorf("standardOpenAPIOperationStore.parameterize() failure; missing required path parameter '%s'", name) } } else if p.Value.In == openapi3.ParameterInQuery { queryParamsRemaining, err := inputParams.GetRemainingQueryParamsFlatMap(copyParams) @@ -1202,7 +1261,11 @@ func (op *standardOperationStore) Parameterize(prov Provider, parentDoc Service, q.Set(k, fmt.Sprintf("%v", v)) delete(copyParams, k) } - router, err := queryrouter.NewRouter(parentDoc.GetT()) + openapiSvc, openapiSvcOk := op.OpenAPIService.(OpenAPIService) + if !openapiSvcOk { + return nil, fmt.Errorf("could not cast OpenAPIService to standardOpenAPIServiceStore") + } + router, err := queryrouter.NewRouter(openapiSvc.getT()) if err != nil { return nil, err } @@ -1268,25 +1331,25 @@ func (op *standardOperationStore) Parameterize(prov Provider, parentDoc Service, return requestValidationInput, nil } -func (op *standardOperationStore) GetRequestBodySchema() (Schema, error) { +func (op *standardOpenAPIOperationStore) GetRequestBodySchema() (Schema, error) { return op.getRequestBodySchema() } -func (op *standardOperationStore) getRequestBodySchema() (Schema, error) { +func (op *standardOpenAPIOperationStore) getRequestBodySchema() (Schema, error) { if op.Request != nil { return op.Request.Schema, nil } return nil, fmt.Errorf("no request body for operation = %s", op.GetName()) } -func (op *standardOperationStore) GetRequestBodyRequiredProperties() ([]string, error) { +func (op *standardOpenAPIOperationStore) GetRequestBodyRequiredProperties() ([]string, error) { if op.Request != nil { return op.Request.Required, nil } return nil, fmt.Errorf("no request body required elements for operation = %s", op.GetName()) } -func (op *standardOperationStore) IsRequiredRequestBodyProperty(key string) bool { +func (op *standardOpenAPIOperationStore) IsRequiredRequestBodyProperty(key string) bool { if op.Request == nil || op.Request.Required == nil { return false } @@ -1298,11 +1361,14 @@ func (op *standardOperationStore) IsRequiredRequestBodyProperty(key string) bool return false } -func (op *standardOperationStore) GetResponseBodySchemaAndMediaType() (Schema, string, error) { +func (op *standardOpenAPIOperationStore) GetResponseBodySchemaAndMediaType() (Schema, string, error) { return op.getResponseBodySchemaAndMediaType() } -func (op *standardOperationStore) getResponseBodySchemaAndMediaType() (Schema, string, error) { +func (op *standardOpenAPIOperationStore) getResponseBodySchemaAndMediaType() (Schema, string, error) { + if op.Response != nil && op.Response.OverrideSchema != nil { + return newSchema(op.Response.OverrideSchema, op.GetService(), "", ""), "", nil + } if op.Response != nil && op.Response.Schema != nil { mediaType := op.Response.BodyMediaType if op.Response.OverrideBodyMediaType != "" { @@ -1313,8 +1379,11 @@ func (op *standardOperationStore) getResponseBodySchemaAndMediaType() (Schema, s return nil, "", fmt.Errorf("no response body for operation = %s", op.GetName()) } -func (op *standardOperationStore) GetSelectSchemaAndObjectPath() (Schema, string, error) { +func (op *standardOpenAPIOperationStore) GetSelectSchemaAndObjectPath() (Schema, string, error) { k := op.lookupSelectItemsKey() + if op.Response != nil && op.Response.OverrideSchema != nil { + return newSchema(op.Response.OverrideSchema, op.GetService(), "", ""), k, nil + } if op.Response != nil && op.Response.Schema != nil { return op.Response.Schema.getSelectItemsSchema(k, op.getOptimalResponseMediaType()) } @@ -1357,7 +1426,7 @@ func (sor *standardOperationResponse) GetReversal() (HTTPPreparator, bool) { return sor.reversal, sor.reversal != nil } -func (op *standardOperationStore) ProcessResponse(response *http.Response) (ProcessedOperationResponse, error) { +func (op *standardOpenAPIOperationStore) ProcessResponse(response *http.Response) (ProcessedOperationResponse, error) { responseSchema, mediaType, err := op.GetResponseBodySchemaAndMediaType() if err != nil { return nil, err @@ -1394,7 +1463,7 @@ func (op *standardOperationStore) ProcessResponse(response *http.Response) (Proc return newStandardOperationResponse(rv, reversal), err } -func (ops *standardOperationStore) lookupSelectItemsKey() string { +func (ops *standardOpenAPIOperationStore) lookupSelectItemsKey() string { s := ops.getSelectItemsKeySimple() if s != "" { return s @@ -1417,7 +1486,7 @@ func (ops *standardOperationStore) lookupSelectItemsKey() string { return "" } -func (op *standardOperationStore) DeprecatedProcessResponse(response *http.Response) (map[string]interface{}, error) { +func (op *standardOpenAPIOperationStore) DeprecatedProcessResponse(response *http.Response) (map[string]interface{}, error) { responseSchema, _, err := op.GetResponseBodySchemaAndMediaType() if err != nil { return nil, err diff --git a/anysdk/operation_store_test.go b/anysdk/operation_store_test.go index eacfbc7..18711a5 100644 --- a/anysdk/operation_store_test.go +++ b/anysdk/operation_store_test.go @@ -1,14 +1,12 @@ -package anysdk_test +package anysdk import ( "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" - . "github.com/stackql/any-sdk/anysdk" - "github.com/stackql/any-sdk/test/pkg/testutil" "gotest.tools/assert" @@ -26,7 +24,7 @@ func TestPlaceholder(t *testing.T) { res := &http.Response{ Header: http.Header{"Content-Type": []string{"application/json"}}, StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(`{"a": { "b": [ "c" ] } }`)), + Body: io.NopCloser(strings.NewReader(`{"a": { "b": [ "c" ] } }`)), } s := NewStringSchema(nil, "", "") pr, err := s.ProcessHttpResponseTesting(res, "", "", "") @@ -47,9 +45,9 @@ func TestXPathHandle(t *testing.T) { t.Fatalf("Test failed: %v", err) } - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) assert.NilError(t, err) assert.Assert(t, svc != nil) @@ -95,9 +93,9 @@ func TestJSONPathHandle(t *testing.T) { b, err := GetServiceDocBytes(fmt.Sprintf("k8s/%s/services/core_v1.yaml", "v0.1.0")) assert.NilError(t, err) - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) assert.NilError(t, err) assert.Assert(t, svc != nil) @@ -144,9 +142,9 @@ func TestJSONPathHandleEnforcedResponseMediaType(t *testing.T) { b, err := GetServiceDocBytes(fmt.Sprintf("k8s/%s/services/core_v1.yaml", "expt")) assert.NilError(t, err) - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) assert.NilError(t, err) assert.Assert(t, svc != nil) @@ -185,9 +183,9 @@ func TestXMLSchemaInterrogation(t *testing.T) { t.Fatalf("Test failed: %v", err) } - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) assert.NilError(t, err) assert.Assert(t, svc != nil) @@ -229,9 +227,9 @@ func TestVariableHostRouting(t *testing.T) { b, err := GetServiceDocBytes(fmt.Sprintf("k8s/%s/services/core_v1.yaml", "v0.1.0")) assert.NilError(t, err) - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) assert.NilError(t, err) assert.Assert(t, svc != nil) @@ -264,7 +262,7 @@ func TestVariableHostRouting(t *testing.T) { err = params.IngestMap(map[string]interface{}{"cluster_addr": "k8shost"}) assert.NilError(t, err) - rvi, err := ops.Parameterize(dummmyK8sProv, svc, params, nil) + rvi, err := ops.parameterize(dummmyK8sProv, svc, params, nil) assert.NilError(t, err) assert.Assert(t, rvi != nil) @@ -272,7 +270,7 @@ func TestVariableHostRouting(t *testing.T) { err = params.IngestMap(map[string]interface{}{"cluster_addr": "201.0.255.3"}) assert.NilError(t, err) - rvi, err = ops.Parameterize(dummmyK8sProv, svc, params, nil) + rvi, err = ops.parameterize(dummmyK8sProv, svc, params, nil) assert.NilError(t, err) assert.Assert(t, rvi != nil) @@ -294,9 +292,9 @@ func TestVariableHostRoutingFutureProofed(t *testing.T) { b, err := GetServiceDocBytes(fmt.Sprintf("k8s/%s/services/core_v1.yaml", "v0.1.1")) assert.NilError(t, err) - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) assert.NilError(t, err) assert.Assert(t, svc != nil) @@ -329,7 +327,7 @@ func TestVariableHostRoutingFutureProofed(t *testing.T) { err = params.IngestMap(map[string]interface{}{"cluster_addr": "k8shost"}) assert.NilError(t, err) - rvi, err := ops.Parameterize(dummmyK8sProv, svc, params, nil) + rvi, err := ops.parameterize(dummmyK8sProv, svc, params, nil) assert.NilError(t, err) assert.Assert(t, rvi != nil) @@ -337,7 +335,7 @@ func TestVariableHostRoutingFutureProofed(t *testing.T) { err = params.IngestMap(map[string]interface{}{"cluster_addr": "201.0.255.3"}) assert.NilError(t, err) - rvi, err = ops.Parameterize(dummmyK8sProv, svc, params, nil) + rvi, err = ops.parameterize(dummmyK8sProv, svc, params, nil) assert.NilError(t, err) assert.Assert(t, rvi != nil) @@ -359,9 +357,9 @@ func TestMethodLevelVariableHostRoutingFutureProofed(t *testing.T) { b, err := GetServiceDocBytes(fmt.Sprintf("contrivedprovider/%s/services/contrived_service.yaml", "v0.1.0")) assert.NilError(t, err) - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) assert.NilError(t, err) assert.Assert(t, svc != nil) @@ -399,7 +397,7 @@ func TestMethodLevelVariableHostRoutingFutureProofed(t *testing.T) { }) assert.NilError(t, err) - rvi, err := ops.Parameterize(dummmyContrivedProv, svc, params, nil) + rvi, err := ops.parameterize(dummmyContrivedProv, svc, params, nil) assert.NilError(t, err) assert.Assert(t, rvi != nil) @@ -411,7 +409,7 @@ func TestMethodLevelVariableHostRoutingFutureProofed(t *testing.T) { }) assert.NilError(t, err) - rvi, err = ops.Parameterize(dummmyContrivedProv, svc, params, nil) + rvi, err = ops.parameterize(dummmyContrivedProv, svc, params, nil) assert.NilError(t, err) assert.Assert(t, rvi != nil) @@ -433,9 +431,9 @@ func TestStaticHostRouting(t *testing.T) { b, err := GetServiceDocBytes(fmt.Sprintf("googleapis.com/%s/services/cloudresourcemanager-v3.yaml", "v0.1.2")) assert.NilError(t, err) - l := NewLoader() + l := newLoader() - svc, err := l.LoadFromBytes(b) + svc, err := l.loadFromBytes(b) assert.NilError(t, err) assert.Assert(t, svc != nil) @@ -477,7 +475,7 @@ func TestStaticHostRouting(t *testing.T) { err = params.IngestMap(map[string]interface{}{"parent": "organizations/123123123123"}) assert.NilError(t, err) - rvi, err := ops.Parameterize(dummmyGoogleProv, svc, params, nil) + rvi, err := ops.parameterize(dummmyGoogleProv, svc, params, nil) assert.NilError(t, err) assert.Assert(t, rvi != nil) diff --git a/anysdk/params.go b/anysdk/params.go index 4e47b77..5edfbb3 100644 --- a/anysdk/params.go +++ b/anysdk/params.go @@ -12,10 +12,10 @@ var ( type standardParameter struct { openapi3.Parameter - svc Service + svc OpenAPIService } -func NewParameter(param *openapi3.Parameter, svc Service) Addressable { +func NewParameter(param *openapi3.Parameter, svc OpenAPIService) Addressable { return &standardParameter{ *param, svc, @@ -28,10 +28,10 @@ type Params interface { type parameters struct { openapi3.Parameters - svc Service + svc OpenAPIService } -func NewParameters(params openapi3.Parameters, svc Service) Params { +func NewParameters(params openapi3.Parameters, svc OpenAPIService) Params { return parameters{ params, svc, diff --git a/anysdk/provider.go b/anysdk/provider.go index 6cec982..911ca06 100644 --- a/anysdk/provider.go +++ b/anysdk/provider.go @@ -6,6 +6,7 @@ import ( "github.com/getkin/kin-openapi/jsoninfo" "github.com/getkin/kin-openapi/openapi3" "github.com/go-openapi/jsonpointer" + "github.com/stackql/any-sdk/pkg/client" ) var ( @@ -18,6 +19,7 @@ type ResponseKeys struct { } type Provider interface { + GetProtocolType() (client.ClientProtocolType, error) Debug() string GetAuth() (AuthDTO, bool) GetDeleteItemsKey() string @@ -26,7 +28,7 @@ type Provider interface { GetPaginationRequestTokenSemantic() (TokenSemantic, bool) GetPaginationResponseTokenSemantic() (TokenSemantic, bool) GetProviderService(key string) (ProviderService, error) - GetQueryTransposeAlgorithm() string + getQueryTransposeAlgorithm() string GetRequestTranslateAlgorithm() string GetResourcesShallow(serviceKey string) (ResourceRegister, error) GetStackQLConfig() (StackQLConfig, bool) @@ -46,6 +48,7 @@ type standardProvider struct { Name string `json:"name" yaml:"name"` Title string `json:"title" yaml:"title"` Version string `json:"version" yaml:"version"` + ProtocolType string `json:"protocolType" yaml:"protocolType"` Description string `json:"description,omitempty" yaml:"desription,omitempty"` ProviderServices map[string]*standardProviderService `json:"providerServices,omitempty" yaml:"providerServices,omitempty"` StackQLConfig *standardStackQLConfig `json:"config,omitempty" yaml:"config,omitempty"` @@ -70,6 +73,13 @@ func (pr *standardProvider) GetName() string { return pr.Name } +func (sv *standardProvider) GetProtocolType() (client.ClientProtocolType, error) { + if sv.ProtocolType == "" { + return client.ClientProtocolTypeFromString(client.ClientProtocolTypeHTTP) + } + return client.ClientProtocolTypeFromString(sv.ProtocolType) +} + func (pr *standardProvider) GetStackQLConfig() (StackQLConfig, bool) { return pr.StackQLConfig, pr.StackQLConfig != nil } @@ -78,7 +88,7 @@ func (pr *standardProvider) GetDeleteItemsKey() string { return pr.DeleteItemsKey } -func (pr *standardProvider) GetQueryTransposeAlgorithm() string { +func (pr *standardProvider) getQueryTransposeAlgorithm() string { if pr.StackQLConfig == nil || pr.StackQLConfig.QueryTranspose == nil { return "" } diff --git a/anysdk/providerService.go b/anysdk/providerService.go index 936336b..ca05da5 100644 --- a/anysdk/providerService.go +++ b/anysdk/providerService.go @@ -6,6 +6,7 @@ import ( "github.com/getkin/kin-openapi/jsoninfo" "github.com/getkin/kin-openapi/openapi3" + "github.com/stackql/any-sdk/pkg/client" "github.com/stackql/stackql-parser/go/sqltypes" ) @@ -16,13 +17,14 @@ var ( type ProviderService interface { ITable - GetQueryTransposeAlgorithm() string + getQueryTransposeAlgorithm() string GetProvider() (Provider, bool) + GetProtocolType() (client.ClientProtocolType, error) GetService() (Service, error) GetRequestTranslateAlgorithm() string GetResourcesShallow() (ResourceRegister, error) GetPaginationRequestTokenSemantic() (TokenSemantic, bool) - GetPaginationResponseTokenSemantic() (TokenSemantic, bool) + getPaginationResponseTokenSemantic() (TokenSemantic, bool) ConditionIsValid(lhs string, rhs interface{}) bool GetID() string GetServiceFragment(resourceKey string) (Service, error) @@ -37,7 +39,7 @@ type ProviderService interface { getResourcesShallowWithRegistry(registry RegistryAPI) (ResourceRegister, error) getServiceRefRef() string getResourcesRefRef() string - setService(svc Service) + setService(svc Service) bool getServiceWithRegistry(registry RegistryAPI) (Service, error) getServiceDocRef(rr ResourceRegister, rsc Resource) ServiceRef setProvider(provider Provider) @@ -45,17 +47,19 @@ type ProviderService interface { type standardProviderService struct { openapi3.ExtensionProps - ID string `json:"id" yaml:"id"` // Required - Name string `json:"name" yaml:"name"` // Required - Title string `json:"title" yaml:"title"` // Required - Version string `json:"version" yaml:"version"` // Required - Description string `json:"description" yaml:"description"` - Preferred bool `json:"preferred" yaml:"preferred"` - ServiceRef *ServiceRef `json:"service,omitempty" yaml:"service,omitempty"` // will be lazy evaluated - ResourcesRef *ResourcesRef `json:"resources,omitempty" yaml:"resources,omitempty"` // will be lazy evaluated - Provider Provider `json:"-" yaml:"-"` // upwards traversal - StackQLConfig *standardStackQLConfig `json:"config,omitempty" yaml:"config,omitempty"` - Service Service `json:"-" yaml:"-"` + ID string `json:"id" yaml:"id"` // Required + Name string `json:"name" yaml:"name"` // Required + Title string `json:"title" yaml:"title"` // Required + Version string `json:"version" yaml:"version"` // Required + Description string `json:"description" yaml:"description"` + Preferred bool `json:"preferred" yaml:"preferred"` + ProtocolType string `json:"protocolType" yaml:"protocolType"` + ServiceRef *ServiceRef `json:"service,omitempty" yaml:"service,omitempty"` // will be lazy evaluated + ResourcesRef *ResourcesRef `json:"resources,omitempty" yaml:"resources,omitempty"` // will be lazy evaluated + Provider Provider `json:"-" yaml:"-"` // upwards traversal + StackQLConfig *standardStackQLConfig `json:"config,omitempty" yaml:"config,omitempty"` + OpenAPIService OpenAPIService `json:"-" yaml:"-"` + GenericService Service `json:"-" yaml:"-"` } func NewEmptyProviderService() ProviderService { @@ -66,6 +70,17 @@ func (sv *standardProviderService) GetTitle() string { return sv.Title } +func (sv *standardProviderService) GetProtocolType() (client.ClientProtocolType, error) { + if sv.ProtocolType != "" { + return client.ClientProtocolTypeFromString(sv.ProtocolType) + } + prov, provOk := sv.GetProvider() + if provOk { + return prov.GetProtocolType() + } + return client.ClientProtocolTypeFromString(client.ClientProtocolTypeHTTP) +} + func (sv *standardProviderService) GetVersion() string { return sv.Version } @@ -96,8 +111,13 @@ func (sv *standardProviderService) GetID() string { return sv.ID } -func (sv *standardProviderService) setService(svc Service) { - sv.Service = svc +func (sv *standardProviderService) setService(svc Service) bool { + openApiSvc, isOpenApiSvc := svc.(OpenAPIService) + if !isOpenApiSvc { + return false + } + sv.OpenAPIService = openApiSvc + return true } func (sv *standardProviderService) getServiceRefRef() string { @@ -122,7 +142,7 @@ func (sv *standardProviderService) GetProvider() (Provider, bool) { return sv.Provider, sv.Provider != nil } -func (sv *standardProviderService) GetQueryTransposeAlgorithm() string { +func (sv *standardProviderService) getQueryTransposeAlgorithm() string { if sv.StackQLConfig != nil { qt, qtExists := sv.StackQLConfig.GetQueryTranspose() if qtExists { @@ -146,7 +166,7 @@ func (sv *standardProviderService) GetPaginationRequestTokenSemantic() (TokenSem return sv.StackQLConfig.Pagination.RequestToken, true } -func (sv *standardProviderService) GetPaginationResponseTokenSemantic() (TokenSemantic, bool) { +func (sv *standardProviderService) getPaginationResponseTokenSemantic() (TokenSemantic, bool) { if sv.StackQLConfig == nil || sv.StackQLConfig.Pagination == nil || sv.StackQLConfig.Pagination.ResponseToken == nil { return nil, false } @@ -224,13 +244,17 @@ func (ps *standardProviderService) getServiceWithRegistry(registry RegistryAPI) if err != nil { return nil, err } - ps.Service = svc - return ps.Service, nil + openapiSvc, isOpenapiSvc := svc.(OpenAPIService) + if !isOpenapiSvc { + return nil, fmt.Errorf("disallowed type for openapi service '%T'", svc) + } + ps.OpenAPIService = openapiSvc + return ps.OpenAPIService, nil } func (ps *standardProviderService) GetService() (Service, error) { - if ps.Service != nil { - return ps.Service, nil + if ps.OpenAPIService != nil { + return ps.OpenAPIService, nil } if ps.ServiceRef.Value != nil { return ps.ServiceRef.Value, nil @@ -239,11 +263,15 @@ func (ps *standardProviderService) GetService() (Service, error) { if err != nil { return nil, err } - ps.Service = svc - return ps.Service, nil + openApiSvc, isOpenApiSvc := svc.(OpenAPIService) + if !isOpenApiSvc { + return nil, fmt.Errorf("disallowed type for openapi service '%T'", svc) + } + ps.OpenAPIService = openApiSvc + return ps.OpenAPIService, nil } -func (ps *standardProviderService) extractService() (Service, error) { +func (ps *standardProviderService) extractService() (OpenAPIService, error) { if ps.ServiceRef.Value != nil { return ps.ServiceRef.Value, nil } @@ -251,8 +279,12 @@ func (ps *standardProviderService) extractService() (Service, error) { if err != nil { return nil, err } - ps.Service = svc - return ps.Service, nil + openApiSvc, isOpenApiSvc := svc.(OpenAPIService) + if !isOpenApiSvc { + return nil, fmt.Errorf("disallowed type for openapi service '%T'", svc) + } + ps.OpenAPIService = openApiSvc + return ps.OpenAPIService, nil } func (ps *standardProviderService) getServiceDocRef(rr ResourceRegister, rsc Resource) ServiceRef { @@ -297,8 +329,8 @@ func (ps *standardProviderService) GetServiceFragment(resourceKey string) (Servi if err != nil { return nil, err } - ps.Service = svc - return ps.Service, nil + ps.OpenAPIService = svc + return ps.OpenAPIService, nil } func (ps *standardProviderService) PeekServiceFragment(resourceKey string) (Service, bool) { diff --git a/anysdk/refs.go b/anysdk/refs.go index 3ac1de8..52312cc 100644 --- a/anysdk/refs.go +++ b/anysdk/refs.go @@ -11,14 +11,19 @@ import ( ) type OperationRef struct { - Ref string `json:"$ref" yaml:"$ref"` - Value *openapi3.Operation + Ref string `json:"$ref" yaml:"$ref"` + Value *openapi3.Operation + Inline []string `json:"inline" yaml:"inline"` } func (opr OperationRef) ExtractPathItem() string { return opr.extractPathItem() } +func (opr OperationRef) GetInline() []string { + return opr.Inline +} + func (opr OperationRef) extractPathItem() string { s := opr.extractFragment() elems := strings.Split(strings.TrimPrefix(s, "/paths/"), "/") @@ -72,19 +77,32 @@ func (opr OperationRef) extractFragment() string { return extractFragment(opr.Ref) } -type OperationStoreRef struct { +type OpenAPIOperationStoreRef struct { Ref string `json:"$ref" yaml:"$ref"` - Value *standardOperationStore + Value *standardOpenAPIOperationStore } -func (osr *OperationStoreRef) hasValue() bool { +func (osr *OpenAPIOperationStoreRef) hasValue() bool { return osr.Value != nil } -func (osr *OperationStoreRef) extractMethodItem() string { +func (osr *OpenAPIOperationStoreRef) extractMethodItem() string { return extractSuffix(osr.Ref) } +type LocalSchemaRef struct { + Ref string `json:"$ref" yaml:"$ref"` + Value *standardSchema +} + +func (osr *LocalSchemaRef) hasValue() bool { + return osr.Value != nil +} + +func (osr *LocalSchemaRef) getSchema() (*standardSchema, bool) { + return osr.Value, osr.hasValue() +} + type PathItemRef struct { Ref string `json:"$ref" yaml:"$ref"` Value *openapi3.PathItem @@ -126,24 +144,24 @@ func (value OperationRef) JSONLookup(token string) (interface{}, error) { return ptr, err } -var _ jsonpointer.JSONPointable = (*OperationStoreRef)(nil) +var _ jsonpointer.JSONPointable = (*OpenAPIOperationStoreRef)(nil) -func (value *OperationStoreRef) MarshalJSON() ([]byte, error) { +func (value *OpenAPIOperationStoreRef) MarshalJSON() ([]byte, error) { return jsoninfo.MarshalRef(value.Ref, value.Value) } -func (value *OperationStoreRef) UnmarshalJSON(data []byte) error { +func (value *OpenAPIOperationStoreRef) UnmarshalJSON(data []byte) error { return jsoninfo.UnmarshalRef(data, &value.Ref, &value.Value) } -// func (value *OperationStoreRef) Validate(ctx context.Context) error { +// func (value *OpenAPIOperationStoreRef) Validate(ctx context.Context) error { // if v := value.Value; v != nil { // return v.Validate(ctx) // } // return foundUnresolvedRef(value.Ref) // } -func (value OperationStoreRef) JSONLookup(token string) (interface{}, error) { +func (value OpenAPIOperationStoreRef) JSONLookup(token string) (interface{}, error) { if token == "$ref" { return value.Ref, nil } diff --git a/anysdk/registry_test.go b/anysdk/registry_test.go index dc82148..774ae75 100644 --- a/anysdk/registry_test.go +++ b/anysdk/registry_test.go @@ -26,67 +26,67 @@ var ( ) const ( - individualDownloadAllowedRegistryCfgStr string = `{"allowSrcDownload": true }` - pullProvidersRegistryCfgStr string = `{"srcPrefix": "test-src" }` - deprecatedRegistryCfgStr string = `{"srcPrefix": "deprecated-src" }` + individualDownloadAllowedRegistryCfgStr string = `{"allowSrcDownload": true, "verifyConfig": { "nopVerify": true } }` + pullProvidersRegistryCfgStr string = `{"srcPrefix": "test-src", "verifyConfig": { "nopVerify": true } }` + deprecatedRegistryCfgStr string = `{"srcPrefix": "deprecated-src", "verifyConfig": { "nopVerify": true } }` unsignedProvidersRegistryCfgStr string = `{"srcPrefix": "unsigned-src", "verifyConfig": { "nopVerify": true } }` ) func init() { var err error - OpenapiFileRoot, err = fileutil.GetFilePathFromRepositoryRoot("providers") + OpenapiFileRoot, err = fileutil.GetFilePathFromRepositoryRoot("test/registry/src") if err != nil { os.Exit(1) } } -// func TestRegistrySimpleOktaApplicationServiceRead(t *testing.T) { -// execLocalAndRemoteRegistryTests(t, individualDownloadAllowedRegistryCfgStr, execTestRegistrySimpleOktaApplicationServiceRead) -// } +func TestRegistrySimpleOktaApplicationServiceRead(t *testing.T) { + execLocalAndRemoteRegistryTests(t, individualDownloadAllowedRegistryCfgStr, execTestRegistrySimpleOktaApplicationServiceRead) +} -// func TestRegistryIndirectGoogleComputeResourcesJsonRead(t *testing.T) { -// execLocalAndRemoteRegistryTests(t, individualDownloadAllowedRegistryCfgStr, execTestRegistryIndirectGoogleComputeResourcesJsonRead) -// } +func TestRegistryIndirectGoogleComputeResourcesJsonRead(t *testing.T) { + execLocalAndRemoteRegistryTests(t, individualDownloadAllowedRegistryCfgStr, execTestRegistryIndirectGoogleComputeResourcesJsonRead) +} -// func TestRegistryIndirectGoogleComputeServiceSubsetAccess(t *testing.T) { -// execLocalAndRemoteRegistryTests(t, individualDownloadAllowedRegistryCfgStr, execTestRegistryIndirectGoogleComputeServiceSubsetAccess) -// } +func TestRegistryIndirectGoogleComputeServiceSubsetAccess(t *testing.T) { + execLocalAndRemoteRegistryTests(t, individualDownloadAllowedRegistryCfgStr, execTestRegistryIndirectGoogleComputeServiceSubsetAccess) +} -// func TestLocalRegistryIndirectGoogleComputeServiceSubsetAccess(t *testing.T) { -// execLocalAndRemoteRegistryTests(t, individualDownloadAllowedRegistryCfgStr, execTestRegistryIndirectGoogleComputeServiceSubsetAccess) -// } +func TestLocalRegistryIndirectGoogleComputeServiceSubsetAccess(t *testing.T) { + execLocalAndRemoteRegistryTests(t, individualDownloadAllowedRegistryCfgStr, execTestRegistryIndirectGoogleComputeServiceSubsetAccess) +} -// func TestProviderPull(t *testing.T) { -// execLocalAndRemoteRegistryTests(t, pullProvidersRegistryCfgStr, execTestRegistrySimpleOktaPull) -// } +func TestProviderPull(t *testing.T) { + execLocalAndRemoteRegistryTests(t, pullProvidersRegistryCfgStr, execTestRegistrySimpleOktaPull) +} -// func TestProviderPullAndPersist(t *testing.T) { -// execLocalAndRemoteRegistryTests(t, pullProvidersRegistryCfgStr, execTestRegistrySimpleOktaPullAndPersist) -// } +func TestProviderPullAndPersist(t *testing.T) { + execRemoteRegistryTestOnly(t, pullProvidersRegistryCfgStr, execTestRegistrySimpleOktaPullAndPersist) +} -// func TestRegistryIndirectGoogleComputeServiceMethodResolutionSeparateDocs(t *testing.T) { -// execLocalRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryIndirectGoogleComputeServiceMethodResolutionSeparateDocs) -// } +func TestRegistryIndirectGoogleComputeServiceMethodResolutionSeparateDocs(t *testing.T) { + execLocalRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryIndirectGoogleComputeServiceMethodResolutionSeparateDocs) +} -// func TestRegistryArrayTopLevelResponse(t *testing.T) { -// execLocalRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryCanHandleArrayResponts) -// } +func TestRegistryArrayTopLevelResponse(t *testing.T) { + execLocalRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryCanHandleArrayResponts) +} -// func TestRegistryCanHandleUnspecifiedResponseWithDefaults(t *testing.T) { -// execLocalRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryCanHandleUnspecifiedResponseWithDefaults) -// } +func TestRegistryCanHandleUnspecifiedResponseWithDefaults(t *testing.T) { + execLocalRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryCanHandleUnspecifiedResponseWithDefaults) +} -// func TestRegistryCanHandlePolymorphismAllOf(t *testing.T) { -// execLocalRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryCanHandlePolymorphismAllOf) -// } +func TestRegistryCanHandlePolymorphismAllOf(t *testing.T) { + execLocalRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryCanHandlePolymorphismAllOf) +} -// func TestListProvidersRegistry(t *testing.T) { -// execRemoteRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryProvidersList) -// } +func TestListProvidersRegistry(t *testing.T) { + execRemoteRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryProvidersList) +} -// func TestListProviderVersionsRegistry(t *testing.T) { -// execRemoteRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryProviderVersionsList) -// } +func TestListProviderVersionsRegistry(t *testing.T) { + execRemoteRegistryTestOnly(t, unsignedProvidersRegistryCfgStr, execTestRegistryProviderVersionsList) +} func execLocalAndRemoteRegistryTests(t *testing.T, registryConfigStr string, tf func(t *testing.T, r RegistryAPI)) { @@ -557,29 +557,93 @@ func execTestRegistryCanHandlePolymorphismAllOf(t *testing.T, r RegistryAPI) { } -// func TestRegistryProviderLatestVersion(t *testing.T) { - -// rc, err := getRegistryCfgFromString(individualDownloadAllowedRegistryCfgStr) -// assert.NilError(t, err) -// r, err := GetMockLocalRegistry(rc) -// assert.NilError(t, err) -// v, err := r.GetLatestAvailableVersion("google") -// assert.NilError(t, err) -// assert.Equal(t, v, "v0.1.2") -// vo, err := r.GetLatestAvailableVersion("okta") -// assert.NilError(t, err) -// assert.Equal(t, vo, "v0.1.0") - -// rc, err = getRegistryCfgFromString(deprecatedRegistryCfgStr) -// assert.NilError(t, err) -// r, err = GetMockLocalRegistry(rc) -// assert.NilError(t, err) -// v, err = r.GetLatestAvailableVersion("google") -// assert.NilError(t, err) -// assert.Equal(t, v, "v1") -// vo, err = r.GetLatestAvailableVersion("okta") -// assert.NilError(t, err) -// assert.Equal(t, vo, "v1") - -// t.Logf("TestRegistryProviderLatestVersion passed\n") -// } +func TestRegistryLocalTemplated(t *testing.T) { + execLocalRegistryTestOnly(t, individualDownloadAllowedRegistryCfgStr, execTestRegistryLocalTemplated) +} + +func execTestRegistryLocalTemplated(t *testing.T, r RegistryAPI) { + + for _, vr := range []string{"v0.1.0"} { + pr, err := r.LoadProviderByName("local_openssl", vr) + if err != nil { + t.Fatalf("Test failed: %v", err) + } + + sh, err := pr.GetProviderService("keys") + + if err != nil { + t.Fatalf("Test failed: %v", err) + } + + assert.Assert(t, sh != nil) + + sv, err := r.GetServiceFragment(sh, "keys") + + assert.NilError(t, err) + + assert.Assert(t, sv != nil) + + // sn := sv.GetName() + + // assert.Equal(t, sn, "repos") + + rsc, err := sv.GetResource("rsa") + + assert.NilError(t, err) + + method, ok := rsc.GetMethods().FindMethod("create_key_pair") + + assert.Assert(t, ok) + + assert.Assert(t, method != nil) + + // assert.Equal(t, os.GetOperationRef().Value.OperationID, "apps/create-from-manifest") + + // assert.Equal(t, os.GetOperationRef().Value.Responses["201"].Value.Content["application/json"].Schema.Value.Type, "") + + // sVal := NewTestSchema(os.GetOperationRef().Value.Responses["201"].Value.Content["application/json"].Schema.Value, sv, "", os.GetOperationRef().Value.Responses["201"].Value.Content["application/json"].Schema.Ref) + + // tab := sVal.Tabulate(false, "") + + // colz := tab.GetColumns() + + // for _, expectedProperty := range []string{"pem", "description"} { + // found := false + // for _, col := range colz { + // if col.GetName() == expectedProperty { + // found = true + // break + // } + // } + // assert.Assert(t, found) + // } + } + +} + +func TestRegistryProviderLatestVersion(t *testing.T) { + + rc, err := getRegistryCfgFromString(individualDownloadAllowedRegistryCfgStr) + assert.NilError(t, err) + r, err := GetMockLocalRegistry(rc) + assert.NilError(t, err) + v, err := r.GetLatestAvailableVersion("google") + assert.NilError(t, err) + assert.Equal(t, v, "v0.1.2") + vo, err := r.GetLatestAvailableVersion("okta") + assert.NilError(t, err) + assert.Equal(t, vo, "v0.1.0") + + rc, err = getRegistryCfgFromString(deprecatedRegistryCfgStr) + assert.NilError(t, err) + r, err = GetMockLocalRegistry(rc) + assert.NilError(t, err) + v, err = r.GetLatestAvailableVersion("google") + assert.NilError(t, err) + assert.Equal(t, v, "v1") + vo, err = r.GetLatestAvailableVersion("okta") + assert.NilError(t, err) + assert.Equal(t, vo, "v1") + + t.Logf("TestRegistryProviderLatestVersion passed\n") +} diff --git a/anysdk/request.go b/anysdk/request.go index d686473..796f8a0 100644 --- a/anysdk/request.go +++ b/anysdk/request.go @@ -9,6 +9,7 @@ import ( "net/http" "github.com/sirupsen/logrus" + "github.com/stackql/any-sdk/pkg/client" "github.com/stackql/any-sdk/pkg/streaming" ) @@ -65,6 +66,10 @@ func newHTTPPreparator( //nolint:funlen,gocognit // TODO: review func (pr *standardHTTPPreparator) BuildHTTPRequestCtx() (HTTPArmoury, error) { + method, methodOk := pr.m.(StandardOperationStore) + if !methodOk { + return nil, fmt.Errorf("operation store is not a standard operation store") + } var err error httpArmoury := NewHTTPArmoury() var requestSchema, responseSchema Schema @@ -78,7 +83,7 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtx() (HTTPArmoury, error) { } httpArmoury.SetRequestSchema(requestSchema) httpArmoury.SetResponseSchema(responseSchema) - paramList, err := splitHTTPParameters(pr.paramMap, pr.m) + paramList, err := splitHTTPParameters(pr.paramMap, method) if err != nil { return nil, err } @@ -94,7 +99,7 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtx() (HTTPArmoury, error) { params.SetRequestBody(pr.execContext.GetExecPayload().GetPayloadMap()) } else if params.GetRequestBody() != nil && len(params.GetRequestBody()) != 0 { m := make(map[string]interface{}) - baseRequestBytes := pr.m.getBaseRequestBodyBytes() + baseRequestBytes := method.getBaseRequestBodyBytes() if len(baseRequestBytes) > 0 { mapErr := json.Unmarshal(baseRequestBytes, &m) if mapErr != nil { @@ -113,14 +118,14 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtx() (HTTPArmoury, error) { if reqExists { pm.SetHeaderKV("Content-Type", []string{req.GetBodyMediaType()}) } - } else if len(pr.m.getDefaultRequestBodyBytes()) > 0 { - pm.SetBodyBytes(pr.m.getDefaultRequestBodyBytes()) - req, reqExists := pr.m.GetRequest() //nolint:govet // intentional shadowing + } else if len(method.getDefaultRequestBodyBytes()) > 0 { + pm.SetBodyBytes(method.getDefaultRequestBodyBytes()) + req, reqExists := method.GetRequest() //nolint:govet // intentional shadowing if reqExists { pm.SetHeaderKV("Content-Type", []string{req.GetBodyMediaType()}) } } - resp, respExists := pr.m.GetResponse() + resp, respExists := method.GetResponse() if respExists { if resp.GetBodyMediaType() != "" && pr.prov.GetName() != "aws" { pm.SetHeaderKV("Accept", []string{resp.GetBodyMediaType()}) @@ -132,12 +137,12 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtx() (HTTPArmoury, error) { secondPassParams := httpArmoury.GetRequestParams() for i, param := range secondPassParams { p := param - if len(p.GetParameters().GetRequestBody()) == 0 && len(pr.m.getDefaultRequestBodyBytes()) == 0 { + if len(p.GetParameters().GetRequestBody()) == 0 && len(method.getDefaultRequestBodyBytes()) == 0 { p.SetRequestBodyMap(nil) - } else if len(pr.m.getDefaultRequestBodyBytes()) > 0 && len(p.GetParameters().GetRequestBody()) == 0 { + } else if len(method.getDefaultRequestBodyBytes()) > 0 && len(p.GetParameters().GetRequestBody()) == 0 { bm := make(map[string]interface{}) // TODO: support types other than json - err := json.Unmarshal(pr.m.getDefaultRequestBodyBytes(), &bm) + err := json.Unmarshal(method.getDefaultRequestBodyBytes(), &bm) if err == nil { p.SetRequestBodyMap(bm) } @@ -193,7 +198,7 @@ func getRequest( if err != nil { return nil, err } - validationParams, err := method.Parameterize(prov, svc, httpParams, httpParams.GetRequestBody()) + validationParams, err := method.parameterize(prov, svc, httpParams, httpParams.GetRequestBody()) if err != nil { return nil, err } @@ -208,6 +213,10 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtxFromAnnotation() (HTTPArmou var err error httpArmoury := NewHTTPArmoury() var requestSchema, responseSchema Schema + httpMethod, httpMethodOk := pr.m.(StandardOperationStore) + if !httpMethodOk { + return nil, fmt.Errorf("operation store is not an http method") + } req, reqExists := pr.m.GetRequest() if reqExists && req.GetSchema() != nil { requestSchema = req.GetSchema() @@ -234,7 +243,7 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtxFromAnnotation() (HTTPArmou return nil, oErr } } - paramList, err := splitHTTPParameters(paramMap, pr.m) + paramList, err := splitHTTPParameters(paramMap, httpMethod) if err != nil { return nil, err } @@ -249,7 +258,7 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtxFromAnnotation() (HTTPArmou params.SetRequestBody(pr.execContext.GetExecPayload().GetPayloadMap()) } else if params.GetRequestBody() != nil && len(params.GetRequestBody()) != 0 { m := make(map[string]interface{}) - baseRequestBytes := pr.m.getBaseRequestBodyBytes() + baseRequestBytes := httpMethod.getBaseRequestBodyBytes() if len(baseRequestBytes) > 0 { mapErr := json.Unmarshal(baseRequestBytes, &m) if mapErr != nil { @@ -268,8 +277,8 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtxFromAnnotation() (HTTPArmou if reqExists { pm.SetHeaderKV("Content-Type", []string{req.GetBodyMediaType()}) } - } else if len(pr.m.getDefaultRequestBodyBytes()) > 0 { - pm.SetBodyBytes(pr.m.getDefaultRequestBodyBytes()) + } else if len(httpMethod.getDefaultRequestBodyBytes()) > 0 { + pm.SetBodyBytes(httpMethod.getDefaultRequestBodyBytes()) req, reqExists := pr.m.GetRequest() //nolint:govet // intentional shadowing if reqExists { pm.SetHeaderKV("Content-Type", []string{req.GetBodyMediaType()}) @@ -287,34 +296,40 @@ func (pr *standardHTTPPreparator) BuildHTTPRequestCtxFromAnnotation() (HTTPArmou secondPassParams := httpArmoury.GetRequestParams() for i, param := range secondPassParams { p := param - if len(p.GetParameters().GetRequestBody()) == 0 && len(pr.m.getDefaultRequestBodyBytes()) == 0 { + if len(p.GetParameters().GetRequestBody()) == 0 && len(httpMethod.getDefaultRequestBodyBytes()) == 0 { p.SetRequestBodyMap(nil) - } else if len(pr.m.getDefaultRequestBodyBytes()) > 0 { + } else if len(httpMethod.getDefaultRequestBodyBytes()) > 0 { bm := make(map[string]interface{}) // TODO: support types other than json - err := json.Unmarshal(pr.m.getDefaultRequestBodyBytes(), &bm) + err := json.Unmarshal(httpMethod.getDefaultRequestBodyBytes(), &bm) if err == nil { p.SetRequestBodyMap(bm) } } var baseRequestCtx *http.Request - baseRequestCtx, err = getRequest(pr.prov, pr.svc, pr.m, p.GetParameters()) - if err != nil { - return nil, err + protocolType, protocolTypeErr := pr.prov.GetProtocolType() + if protocolTypeErr != nil { + return nil, protocolTypeErr } - for k, v := range p.GetHeader() { - for _, vi := range v { - baseRequestCtx.Header.Set(k, vi) + if protocolType == client.HTTP { + baseRequestCtx, err = getRequest(pr.prov, pr.svc, pr.m, p.GetParameters()) + if err != nil { + return nil, err + } + for k, v := range p.GetHeader() { + for _, vi := range v { + baseRequestCtx.Header.Set(k, vi) + } } - } - p.SetRequest(baseRequestCtx) - pr.logger.Infoln( - fmt.Sprintf("pre transform: httpArmoury.RequestParams[%d] = %s", - i, string(p.GetBodyBytes()))) - pr.logger.Infoln( - fmt.Sprintf("post transform: httpArmoury.RequestParams[%d] = %s", - i, string(p.GetBodyBytes()))) + p.SetRequest(baseRequestCtx) + pr.logger.Infoln( + fmt.Sprintf("pre transform: httpArmoury.RequestParams[%d] = %s", + i, string(p.GetBodyBytes()))) + pr.logger.Infoln( + fmt.Sprintf("post transform: httpArmoury.RequestParams[%d] = %s", + i, string(p.GetBodyBytes()))) + } secondPassParams[i] = p } httpArmoury.SetRequestParams(secondPassParams) diff --git a/anysdk/resource.go b/anysdk/resource.go index c4b5b96..7b9a946 100644 --- a/anysdk/resource.go +++ b/anysdk/resource.go @@ -16,7 +16,7 @@ var ( type Resource interface { ITable - GetQueryTransposeAlgorithm() string + getQueryTransposeAlgorithm() string GetID() string GetTitle() string GetDescription() string @@ -26,44 +26,44 @@ type Resource interface { GetRequestTranslateAlgorithm() string GetPaginationRequestTokenSemantic() (TokenSemantic, bool) GetPaginationResponseTokenSemantic() (TokenSemantic, bool) - FindMethod(key string) (OperationStore, error) - GetFirstMethodFromSQLVerb(sqlVerb string) (OperationStore, string, bool) - GetFirstMethodMatchFromSQLVerb(sqlVerb string, parameters map[string]interface{}) (OperationStore, map[string]interface{}, bool) - GetService() (Service, bool) + FindMethod(key string) (StandardOperationStore, error) + GetFirstMethodFromSQLVerb(sqlVerb string) (StandardOperationStore, string, bool) + GetFirstMethodMatchFromSQLVerb(sqlVerb string, parameters map[string]interface{}) (StandardOperationStore, map[string]interface{}, bool) + GetService() (OpenAPIService, bool) GetViewsForSqlDialect(sqlDialect string) ([]View, bool) GetMethodsMatched() Methods ToMap(extended bool) map[string]interface{} // unexported mutators - getSQLVerbs() map[string][]OperationStoreRef + getSQLVerbs() map[string][]OpenAPIOperationStoreRef setProvider(p Provider) - setService(s Service) + setService(s OpenAPIService) setProviderService(ps ProviderService) - getUnionRequiredParameters(method OperationStore) (map[string]Addressable, error) - setMethod(string, *standardOperationStore) - mutateSQLVerb(k string, idx int, v OperationStoreRef) + getUnionRequiredParameters(method StandardOperationStore) (map[string]Addressable, error) + setMethod(string, *standardOpenAPIOperationStore) + mutateSQLVerb(k string, idx int, v OpenAPIOperationStoreRef) propogateToConfig() error } type standardResource struct { - ID string `json:"id" yaml:"id"` // Required - Name string `json:"name" yaml:"name"` // Required - Title string `json:"title" yaml:"title"` // Required - Description string `json:"description,omitempty" yaml:"desription,omitempty"` - SelectorAlgorithm string `json:"selectorAlgorithm,omitempty" yaml:"selectorAlgorithm,omitempty"` - Methods Methods `json:"methods" yaml:"methods"` - ServiceDocPath *ServiceRef `json:"serviceDoc,omitempty" yaml:"serviceDoc,omitempty"` - SQLVerbs map[string][]OperationStoreRef `json:"sqlVerbs" yaml:"sqlVerbs"` - BaseUrl string `json:"baseUrl,omitempty" yaml:"baseUrl,omitempty"` // hack - StackQLConfig *standardStackQLConfig `json:"config,omitempty" yaml:"config,omitempty"` - Service Service `json:"-" yaml:"-"` // upwards traversal - ProviderService ProviderService `json:"-" yaml:"-"` // upwards traversal - Provider Provider `json:"-" yaml:"-"` // upwards traversal + ID string `json:"id" yaml:"id"` // Required + Name string `json:"name" yaml:"name"` // Required + Title string `json:"title" yaml:"title"` // Required + Description string `json:"description,omitempty" yaml:"desription,omitempty"` + SelectorAlgorithm string `json:"selectorAlgorithm,omitempty" yaml:"selectorAlgorithm,omitempty"` + Methods Methods `json:"methods" yaml:"methods"` + ServiceDocPath *ServiceRef `json:"serviceDoc,omitempty" yaml:"serviceDoc,omitempty"` + SQLVerbs map[string][]OpenAPIOperationStoreRef `json:"sqlVerbs" yaml:"sqlVerbs"` + BaseUrl string `json:"baseUrl,omitempty" yaml:"baseUrl,omitempty"` // hack + StackQLConfig *standardStackQLConfig `json:"config,omitempty" yaml:"config,omitempty"` + OpenAPIService OpenAPIService `json:"-" yaml:"-"` // upwards traversal + ProviderService ProviderService `json:"-" yaml:"-"` // upwards traversal + Provider Provider `json:"-" yaml:"-"` // upwards traversal } func NewEmptyResource() Resource { return &standardResource{ Methods: make(Methods), - SQLVerbs: make(map[string][]OperationStoreRef), + SQLVerbs: make(map[string][]OpenAPIOperationStoreRef), } } @@ -75,26 +75,26 @@ func (r *standardResource) propogateToConfig() error { return nil } -func (r *standardResource) GetService() (Service, bool) { - if r.Service == nil { +func (r *standardResource) GetService() (OpenAPIService, bool) { + if r.OpenAPIService == nil { return nil, false } - return r.Service, true + return r.OpenAPIService, true } -func (r *standardResource) getSQLVerbs() map[string][]OperationStoreRef { +func (r *standardResource) getSQLVerbs() map[string][]OpenAPIOperationStoreRef { return r.SQLVerbs } -func (r *standardResource) setService(s Service) { - r.Service = s +func (r *standardResource) setService(s OpenAPIService) { + r.OpenAPIService = s } -func (r *standardResource) mutateSQLVerb(k string, idx int, v OperationStoreRef) { +func (r *standardResource) mutateSQLVerb(k string, idx int, v OpenAPIOperationStoreRef) { r.SQLVerbs[k][idx] = v } -func (r *standardResource) setMethod(k string, v *standardOperationStore) { +func (r *standardResource) setMethod(k string, v *standardOpenAPIOperationStore) { if v == nil { return } @@ -133,7 +133,7 @@ func (r *standardResource) GetServiceDocPath() *ServiceRef { return r.ServiceDocPath } -func (r *standardResource) GetQueryTransposeAlgorithm() string { +func (r *standardResource) getQueryTransposeAlgorithm() string { if r.StackQLConfig == nil || r.StackQLConfig.QueryTranspose == nil { return "" } @@ -191,7 +191,7 @@ func (rsc standardResource) JSONLookup(token string) (interface{}, error) { } return &m, nil default: - val, _, err := jsonpointer.GetForToken(rsc.Service.GetT(), token) + val, _, err := jsonpointer.GetForToken(rsc.OpenAPIService.getT(), token) return val, err } } @@ -235,11 +235,11 @@ func (rs *standardResource) getMethodsMatched() Methods { return rv } -func (rs *standardResource) GetFirstMethodMatchFromSQLVerb(sqlVerb string, parameters map[string]interface{}) (OperationStore, map[string]interface{}, bool) { +func (rs *standardResource) GetFirstMethodMatchFromSQLVerb(sqlVerb string, parameters map[string]interface{}) (StandardOperationStore, map[string]interface{}, bool) { return rs.getFirstMethodMatchFromSQLVerb(sqlVerb, parameters) } -func (rs *standardResource) getFirstMethodMatchFromSQLVerb(sqlVerb string, parameters map[string]interface{}) (OperationStore, map[string]interface{}, bool) { +func (rs *standardResource) getFirstMethodMatchFromSQLVerb(sqlVerb string, parameters map[string]interface{}) (StandardOperationStore, map[string]interface{}, bool) { ms, err := rs.getMethodsForSQLVerb(sqlVerb) if err != nil { return nil, parameters, false @@ -247,11 +247,11 @@ func (rs *standardResource) getFirstMethodMatchFromSQLVerb(sqlVerb string, param return ms.getFirstMatch(parameters) } -func (rs *standardResource) GetFirstMethodFromSQLVerb(sqlVerb string) (OperationStore, string, bool) { +func (rs *standardResource) GetFirstMethodFromSQLVerb(sqlVerb string) (StandardOperationStore, string, bool) { return rs.getFirstMethodFromSQLVerb(sqlVerb) } -func (rs *standardResource) getUnionRequiredParameters(method OperationStore) (map[string]Addressable, error) { +func (rs *standardResource) getUnionRequiredParameters(method StandardOperationStore) (map[string]Addressable, error) { targetSchema, _, err := method.GetSelectSchemaAndObjectPath() if err != nil { return nil, fmt.Errorf("getUnionRequiredParameters(): cannot infer fat required parameters: %s", err.Error()) @@ -281,7 +281,7 @@ func (rs *standardResource) getUnionRequiredParameters(method OperationStore) (m return rv, nil } -func (rs *standardResource) getFirstMethodFromSQLVerb(sqlVerb string) (OperationStore, string, bool) { +func (rs *standardResource) getFirstMethodFromSQLVerb(sqlVerb string) (StandardOperationStore, string, bool) { ms, err := rs.getMethodsForSQLVerb(sqlVerb) if err != nil { return nil, "", false @@ -352,15 +352,15 @@ func (rs *standardResource) GetSelectableObject() string { return "" } -func (rs *standardResource) FindOperationStore(sel OperationSelector) (OperationStore, error) { +func (rs *standardResource) FindOpenAPIOperationStore(sel OperationSelector) (StandardOperationStore, error) { switch rs.SelectorAlgorithm { case "", "standard": - return rs.findOperationStoreStandard(sel) + return rs.findOpenAPIOperationStoreStandard(sel) } return nil, fmt.Errorf("cannot search for operation with selector algorithm = '%s'", rs.SelectorAlgorithm) } -func (rs *standardResource) findOperationStoreStandard(sel OperationSelector) (OperationStore, error) { +func (rs *standardResource) findOpenAPIOperationStoreStandard(sel OperationSelector) (StandardOperationStore, error) { rv, err := rs.Methods.FindFromSelector(sel) if err == nil { return rv, nil @@ -377,7 +377,7 @@ func (r *standardResource) FilterBy(predicate func(interface{}) (ITable, error)) return predicate(r) } -func (r *standardResource) FindMethod(key string) (OperationStore, error) { +func (r *standardResource) FindMethod(key string) (StandardOperationStore, error) { if r.Methods == nil { return nil, fmt.Errorf("cannot find method with key = '%s' from nil methods", key) } diff --git a/anysdk/resourceRegister.go b/anysdk/resourceRegister.go index 8f5e5f8..f8aadad 100644 --- a/anysdk/resourceRegister.go +++ b/anysdk/resourceRegister.go @@ -15,7 +15,7 @@ type ResourceRegister interface { // getProvider() Provider getProviderService() ProviderService - setOpStore(resourceString string, methodString string, opStore OperationStore) + setOpStore(resourceString string, methodString string, opStore StandardOperationStore) } func NewResourceRegister() ResourceRegister { @@ -41,8 +41,8 @@ func (rr *standardResourceRegister) GetResource(resourceKey string) (Resource, b return rsc, ok } -func (rr *standardResourceRegister) setOpStore(resourceString string, methodString string, opStore OperationStore) { - rr.Resources[resourceString].setMethod(methodString, opStore.(*standardOperationStore)) +func (rr *standardResourceRegister) setOpStore(resourceString string, methodString string, opStore StandardOperationStore) { + rr.Resources[resourceString].setMethod(methodString, opStore.(*standardOpenAPIOperationStore)) } func (rr *standardResourceRegister) getProvider() Provider { diff --git a/anysdk/schema.go b/anysdk/schema.go index 8b0142e..9dd212e 100644 --- a/anysdk/schema.go +++ b/anysdk/schema.go @@ -74,7 +74,7 @@ type Schema interface { setItemsRef(*openapi3.SchemaRef) setPropertyOpenapi3(k string, ps *openapi3.SchemaRef) getPropertiesColumns() []ColumnDescriptor - getService() Service + getService() OpenAPIService getFatSchema(srs openapi3.SchemaRefs) Schema getXml() (interface{}, bool) getXmlAlias() string @@ -232,7 +232,7 @@ func (s *standardSchema) getOpenapiSchema() (*openapi3.Schema, bool) { type standardSchema struct { *openapi3.Schema - svc Service + svc OpenAPIService key string alwaysRequired bool path string @@ -240,7 +240,7 @@ type standardSchema struct { defaultColName string } -func (s *standardSchema) getService() Service { +func (s *standardSchema) getService() OpenAPIService { return s.svc } @@ -296,14 +296,25 @@ func copyOpenapiSchema(inSchema *openapi3.Schema) *openapi3.Schema { type Schemas map[string]Schema -func NewStringSchema(svc Service, key string, path string) Schema { +func NewStringSchema(svc OpenAPIService, key string, path string) Schema { sc := openapi3.NewSchema() sc.Type = "string" return newSchema(sc, svc, key, path) } +func newExmptyObjectStandardSchema(svc OpenAPIService, key string, path string) *standardSchema { + sc := openapi3.NewSchema() + sc.Type = "object" + sc.Properties = openapi3.Schemas{} + return newStandardSchema(sc, svc, key, path) +} + func NewTestSchema(sc *openapi3.Schema, svc Service, key string, path string) Schema { - return newSchema(sc, svc, key, path) + openapiSvc, ok := svc.(OpenAPIService) + if !ok { + panic("NewTestSchema() requires an OpenAPIService") + } + return newSchema(sc, openapiSvc, key, path) } func (sc *standardSchema) GetPath() string { @@ -317,7 +328,16 @@ func (sc *standardSchema) GetAdditionalProperties() (Schema, bool) { return newSchema(sc.AdditionalProperties.Value, sc.svc, "additionalProperties", sc.AdditionalProperties.Ref), true } -func newSchema(sc *openapi3.Schema, svc Service, key string, path string) Schema { +func newSchema(sc *openapi3.Schema, svc OpenAPIService, key string, path string) Schema { + return newStandardSchema( + sc, + svc, + key, + path, + ) +} + +func newStandardSchema(sc *openapi3.Schema, svc OpenAPIService, key string, path string) *standardSchema { var alwaysRequired bool if sc.Extensions != nil { if ar, ok := sc.Extensions[ExtensionKeyAlwaysRequired]; ok { diff --git a/anysdk/server.go b/anysdk/server.go index eb2ef73..ef7ad1a 100644 --- a/anysdk/server.go +++ b/anysdk/server.go @@ -12,7 +12,7 @@ func ObtainServerURLsFromServers(svs []*openapi3.Server, vars map[string]string) return obtainServerURLsFromServers(svs, vars) } -func getServerVariablesMap(sv *openapi3.Server, svc Service) map[string]Addressable { +func getServerVariablesMap(sv *openapi3.Server, svc OpenAPIService) map[string]Addressable { retVal := make(map[string]Addressable) for k, v := range sv.Variables { s := openapi3.NewSchema() diff --git a/anysdk/service.go b/anysdk/service.go index c25315f..1ae2f93 100644 --- a/anysdk/service.go +++ b/anysdk/service.go @@ -12,24 +12,28 @@ import ( ) var ( - _ Service = &standardService{} + _ Service = &localTemplatedService{} + _ OpenAPIService = &standardService{} ) type Service interface { - GetT() *openapi3.T - GetQueryTransposeAlgorithm() string IsPreferred() bool - GetRequestTranslateAlgorithm() string - GetPaginationRequestTokenSemantic() (TokenSemantic, bool) - GetPaginationResponseTokenSemantic() (TokenSemantic, bool) - GetServers() (openapi3.Servers, bool) + GetServers() (openapi3.Servers, bool) // Difficult to remove, not impossible. GetResources() (map[string]Resource, error) - GetComponents() openapi3.Components GetName() string GetResource(resourceName string) (Resource, error) GetSchema(key string) (Schema, error) - GetContactURL() string +} + +type OpenAPIService interface { + Service // + getRequestTranslateAlgorithm() string + getComponents() openapi3.Components + getPaginationRequestTokenSemantic() (TokenSemantic, bool) + getPaginationResponseTokenSemantic() (TokenSemantic, bool) + getQueryTransposeAlgorithm() string + getT() *openapi3.T iDiscoveryDoc() isObjectSchemaImplicitlyUnioned() bool getExtension(key string) (interface{}, bool) @@ -42,6 +46,50 @@ type Service interface { getPath(k string) (*openapi3.PathItem, bool) } +type localTemplatedService struct { + Name string `json:"name" yaml:"name"` + Rsc map[string]*standardResource `json:"resources" yaml:"resources"` + StackQLConfig StackQLConfig `json:"-" yaml:"-"` + ProviderService ProviderService `json:"-" yaml:"-"` // upwards traversal + Provider Provider `json:"-" yaml:"-"` // upwards traversal +} + +func (sv *localTemplatedService) GetServers() (openapi3.Servers, bool) { + return nil, false +} + +func (sv *localTemplatedService) IsPreferred() bool { + return true +} + +func (sv *localTemplatedService) GetResources() (map[string]Resource, error) { + rv := make(map[string]Resource) + for k, v := range sv.Rsc { + rv[k] = v + } + return rv, nil +} + +func (sv *localTemplatedService) GetResource(resourceName string) (Resource, error) { + rscs, err := sv.GetResources() + if err != nil { + return nil, err + } + rsc, ok := rscs[resourceName] + if !ok { + return nil, fmt.Errorf("OpenAPIService.GetResource() failure") + } + return rsc, nil +} + +func (sv *localTemplatedService) GetName() string { + return sv.Name +} + +func (sv *localTemplatedService) GetSchema(key string) (Schema, error) { + return nil, fmt.Errorf("GetSchema not implemented for localTemplatedService") +} + type standardService struct { *openapi3.T rsc map[string]*standardResource @@ -50,13 +98,6 @@ type standardService struct { Provider Provider `json:"-" yaml:"-"` // upwards traversal } -func (sv *standardService) GetContactURL() string { - if sv.Info == nil || sv.Info.Contact == nil { - return "" - } - return sv.Info.Contact.URL -} - func (sv *standardService) getPath(k string) (*openapi3.PathItem, bool) { rv, ok := sv.T.Paths[k] return rv, ok @@ -70,7 +111,7 @@ func (sv *standardService) getProvider() Provider { return sv.Provider } -func (sv *standardService) GetComponents() openapi3.Components { +func (sv *standardService) getComponents() openapi3.Components { return sv.T.Components } @@ -82,7 +123,7 @@ func (sv *standardService) setProvider(provider Provider) { for _, m := range rsc.Methods { m.setProvider(provider) if m.Inverse != nil { - inverseOpStore, inverseOpStoreExists := m.Inverse.getOperationStore() + inverseOpStore, inverseOpStoreExists := m.Inverse.getOpenAPIOperationStore() if inverseOpStoreExists { inverseOpStore.setProvider(provider) } @@ -101,7 +142,7 @@ func (sv *standardService) setProviderService(providerService ProviderService) { for _, m := range rsc.Methods { m.setProviderService(providerService) if m.Inverse != nil { - inverseOpStore, inverseOpStoreExists := m.Inverse.getOperationStore() + inverseOpStore, inverseOpStoreExists := m.Inverse.getOpenAPIOperationStore() if inverseOpStoreExists { inverseOpStore.setProviderService(providerService) } @@ -127,7 +168,7 @@ func (sv *standardService) setResourceMap(rsc map[string]*standardResource) { func (sv *standardService) iDiscoveryDoc() {} -func (sv *standardService) GetT() *openapi3.T { +func (sv *standardService) getT() *openapi3.T { return sv.T } @@ -141,7 +182,7 @@ func (sv *standardService) isObjectSchemaImplicitlyUnioned() bool { return sv.Provider.isObjectSchemaImplicitlyUnioned() } -func NewService(t *openapi3.T) Service { +func NewService(t *openapi3.T) OpenAPIService { svc := &standardService{ T: t, rsc: make(map[string]*standardResource), @@ -158,7 +199,7 @@ func (svc *standardService) IsPreferred() bool { return false } -func (svc *standardService) GetQueryTransposeAlgorithm() string { +func (svc *standardService) getQueryTransposeAlgorithm() string { if svc.StackQLConfig != nil { qt, qtExists := svc.StackQLConfig.GetQueryTranspose() if qtExists { @@ -168,7 +209,7 @@ func (svc *standardService) GetQueryTransposeAlgorithm() string { return "" } -func (svc *standardService) GetRequestTranslateAlgorithm() string { +func (svc *standardService) getRequestTranslateAlgorithm() string { if svc.StackQLConfig != nil { rt, rtExists := svc.StackQLConfig.GetRequestTranslate() if rtExists { @@ -178,7 +219,7 @@ func (svc *standardService) GetRequestTranslateAlgorithm() string { return "" } -func (svc *standardService) GetPaginationRequestTokenSemantic() (TokenSemantic, bool) { +func (svc *standardService) getPaginationRequestTokenSemantic() (TokenSemantic, bool) { if svc.StackQLConfig != nil { pag, pagExists := svc.StackQLConfig.GetPagination() if pagExists && pag.GetRequestToken() != nil { @@ -188,7 +229,7 @@ func (svc *standardService) GetPaginationRequestTokenSemantic() (TokenSemantic, return nil, false } -func (svc *standardService) GetPaginationResponseTokenSemantic() (TokenSemantic, bool) { +func (svc *standardService) getPaginationResponseTokenSemantic() (TokenSemantic, bool) { if svc.StackQLConfig != nil { pag, pagExists := svc.StackQLConfig.GetPagination() if pagExists && pag.GetResponseToken() != nil { @@ -307,7 +348,7 @@ func (svc *standardService) GetResource(resourceName string) (Resource, error) { } rsc, ok := rscs[resourceName] if !ok { - return nil, fmt.Errorf("Service.GetResource() failure") + return nil, fmt.Errorf("OpenAPIService.GetResource() failure") } return rsc, nil } diff --git a/anysdk/shims.go b/anysdk/shims.go index fd87578..0728781 100644 --- a/anysdk/shims.go +++ b/anysdk/shims.go @@ -156,7 +156,7 @@ func newObjectWithLineage(val interface{}, schema Schema, parentKey string, path } } -func parseRequestBodyParam(k string, v interface{}, s Schema, method OperationStore) (ObjectWithLineage, bool) { +func parseRequestBodyParam(k string, v interface{}, s Schema, method StandardOperationStore) (ObjectWithLineage, bool) { trimmedKey, revertErr := method.revertRequestBodyAttributeRename(k) var parsedVal interface{} if revertErr == nil { //nolint:nestif // keep for now @@ -210,7 +210,7 @@ func parseRequestBodyParam(k string, v interface{}, s Schema, method OperationSt //nolint:gocognit // not super complex func splitHTTPParameters( sqlParamMap map[int]map[string]interface{}, - method OperationStore, + method StandardOperationStore, ) ([]HttpParameters, error) { var retVal []HttpParameters var rowKeys []int diff --git a/anysdk/table.go b/anysdk/table.go index ba13889..386dbcf 100644 --- a/anysdk/table.go +++ b/anysdk/table.go @@ -2,6 +2,7 @@ package anysdk import ( "github.com/getkin/kin-openapi/openapi3" + "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/stackql-parser/go/sqltypes" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -101,6 +102,14 @@ func newColumnDescriptor(alias string, name string, qualifier string, decoratedC return standardColumnDescriptor{Alias: alias, Name: name, Qualifier: qualifier, DecoratedCol: decoratedCol, Schema: schema, Val: val, Node: node} } +func newNilTabulation(svc OpenAPIService, key string, path string) Tabulation { + return newStandardTabulation( + constants.NilTabulationName, + []ColumnDescriptor{}, + newExmptyObjectStandardSchema(svc, key, path), + ) +} + type Tabulation interface { GetColumns() []ColumnDescriptor GetSchema() Schema diff --git a/anysdk/transform.go b/anysdk/transform.go index 0f2ef8f..bed1add 100644 --- a/anysdk/transform.go +++ b/anysdk/transform.go @@ -14,16 +14,28 @@ var ( type Transform interface { JSONLookup(token string) (interface{}, error) GetAlgorithm() string + GetType() string + GetBody() string } type standardTransform struct { Algorithm string `json:"algorithm,omitempty" yaml:"algorithm,omitempty"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Body string `json:"body,omitempty" yaml:"body,omitempty"` } func (ts standardTransform) GetAlgorithm() string { return ts.Algorithm } +func (ts standardTransform) GetType() string { + return ts.Type +} + +func (ts standardTransform) GetBody() string { + return ts.Body +} + func (qt standardTransform) JSONLookup(token string) (interface{}, error) { switch token { case "algorithm": diff --git a/cmd/argparse/query.go b/cmd/argparse/query.go index ccebae1..09ebf94 100644 --- a/cmd/argparse/query.go +++ b/cmd/argparse/query.go @@ -1,106 +1,26 @@ package argparse import ( + "bytes" "encoding/json" "fmt" "io" - "net/http" "os" "runtime/pprof" - "strings" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" "github.com/stackql/any-sdk/anysdk" - "github.com/stackql/any-sdk/pkg/auth_util" + "github.com/stackql/any-sdk/pkg/client" "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/any-sdk/pkg/internaldto" - "github.com/stackql/any-sdk/pkg/netutils" + "github.com/stackql/any-sdk/pkg/local_template_executor" + "github.com/stackql/any-sdk/pkg/stream_transform" ) -type genericProvider struct { - provider anysdk.Provider - runtimeCtx dto.RuntimeCtx - authUtil auth_util.AuthUtility -} - -func newGenericProvider(rtCtx dto.RuntimeCtx, prov anysdk.Provider) *genericProvider { - return &genericProvider{ - runtimeCtx: rtCtx, - authUtil: auth_util.NewAuthUtility(), - provider: prov, - } -} - -func (gp *genericProvider) inferAuthType(authCtx dto.AuthCtx, authTypeRequested string) string { - ft := strings.ToLower(authTypeRequested) - switch ft { - case dto.AuthAzureDefaultStr: - return dto.AuthAzureDefaultStr - case dto.AuthAPIKeyStr: - return dto.AuthAPIKeyStr - case dto.AuthBasicStr: - return dto.AuthBasicStr - case dto.AuthBearerStr: - return dto.AuthBearerStr - case dto.AuthServiceAccountStr: - return dto.AuthServiceAccountStr - case dto.AuthInteractiveStr: - return dto.AuthInteractiveStr - case dto.AuthNullStr: - return dto.AuthNullStr - case dto.AuthAWSSigningv4Str: - return dto.AuthAWSSigningv4Str - case dto.AuthCustomStr: - return dto.AuthCustomStr - case dto.OAuth2Str: - return dto.OAuth2Str - } - if authCtx.KeyFilePath != "" || authCtx.KeyEnvVar != "" { - return dto.AuthServiceAccountStr - } - return dto.AuthNullStr -} - -func (gp *genericProvider) Auth( - authCtx *dto.AuthCtx, - authTypeRequested string, - enforceRevokeFirst bool, -) (*http.Client, error) { - authCtx = authCtx.Clone() - at := gp.inferAuthType(*authCtx, authTypeRequested) - switch at { - case dto.AuthAPIKeyStr: - return gp.authUtil.ApiTokenAuth(authCtx, gp.runtimeCtx, false) - case dto.AuthBearerStr: - return gp.authUtil.ApiTokenAuth(authCtx, gp.runtimeCtx, true) - case dto.AuthServiceAccountStr: - scopes := authCtx.Scopes - return gp.authUtil.GoogleOauthServiceAccount(gp.provider.GetName(), authCtx, scopes, gp.runtimeCtx) - case dto.OAuth2Str: - if authCtx.GrantType == dto.ClientCredentialsStr { - scopes := authCtx.Scopes - return gp.authUtil.GenericOauthClientCredentials(authCtx, scopes, gp.runtimeCtx) - } - case dto.AuthBasicStr: - return gp.authUtil.BasicAuth(authCtx, gp.runtimeCtx) - case dto.AuthCustomStr: - return gp.authUtil.CustomAuth(authCtx, gp.runtimeCtx) - case dto.AuthAzureDefaultStr: - return gp.authUtil.AzureDefaultAuth(authCtx, gp.runtimeCtx) - case dto.AuthInteractiveStr: - return gp.authUtil.GCloudOAuth(gp.runtimeCtx, authCtx, enforceRevokeFirst) - case dto.AuthAWSSigningv4Str: - return gp.authUtil.AwsSigningAuth(authCtx, gp.runtimeCtx) - case dto.AuthNullStr: - return netutils.GetHTTPClient(gp.runtimeCtx, http.DefaultClient), nil - } - return nil, fmt.Errorf("could not infer auth type") -} - func getLogger() *logrus.Logger { logger := logrus.New() logger.SetOutput(os.Stderr) @@ -147,24 +67,7 @@ type queryCmdPayload struct { } func (qcp *queryCmdPayload) getService() (anysdk.Service, error) { - pb, err := os.ReadFile(qcp.provFilePath) - if err != nil { - return nil, err - } - prov, err := anysdk.LoadProviderDocFromBytes(pb) - if err != nil { - return nil, err - } - b, err := os.ReadFile(qcp.svcFilePath) - if err != nil { - return nil, err - } - l := anysdk.NewLoader() - svc, err := l.LoadFromBytesWithProvider(b, prov) - if err != nil { - return nil, err - } - return svc, nil + return anysdk.LoadProviderAndServiceFromPaths(qcp.provFilePath, qcp.svcFilePath) } func (qcp *queryCmdPayload) getProvider() (anysdk.Provider, error) { @@ -203,7 +106,7 @@ func newQueryCmdPayload(rtCtx dto.RuntimeCtx) (*queryCmdPayload, error) { }, nil } -func runQueryCommand(gp *genericProvider, authCtx *dto.AuthCtx, payload *queryCmdPayload) error { +func runQueryCommand(authCtx *dto.AuthCtx, payload *queryCmdPayload) error { prov, err := payload.getProvider() if err != nil { return err @@ -231,43 +134,100 @@ func runQueryCommand(gp *genericProvider, authCtx *dto.AuthCtx, payload *queryCm execPayload, res, ) - prep := anysdk.NewHTTPPreparator( - prov, - svc, - opStore, - map[int]map[string]interface{}{ - 0: payload.parameters, - }, - nil, - execCtx, - getLogger(), - ) - httpClient, err := gp.Auth( - authCtx, - authCtx.Type, - false, - ) - if err != nil { - return err + protocolType, protocolTypeErr := prov.GetProtocolType() + if protocolTypeErr != nil { + return protocolTypeErr } - armoury, err := prep.BuildHTTPRequestCtx() - if err != nil { - return err - } - for _, v := range armoury.GetRequestParams() { - req := v.GetRequest() - response, err := httpClient.Do(req) + switch protocolType { + case client.LocalTemplated: + inlines := opStore.GetInline() + if len(inlines) == 0 { + return fmt.Errorf("no inlines found") + } + executor := local_template_executor.NewLocalTemplateExecutor( + inlines[0], + inlines[1:], + nil, + ) + resp, err := executor.Execute( + map[string]any{"parameters": payload.parameters}, + ) if err != nil { return err } - defer response.Body.Close() - bodyBytes, err := io.ReadAll(response.Body) + stdOut, stdOutExists := resp.GetStdOut() + stdoutStr := "" + if stdOutExists { + stdoutStr = stdOut.String() + expectedResponse, isExpectedResponse := opStore.GetResponse() + if isExpectedResponse { + responseTransform, responseTransformExists := expectedResponse.GetTransform() + if responseTransformExists && responseTransform.GetType() == "golang_template_v0.1.0" { + input := stdoutStr + tmpl := responseTransform.GetBody() + inStream := stream_transform.NewTextReader(bytes.NewBufferString(input)) + outStream := bytes.NewBuffer(nil) + tfm, err := stream_transform.NewTemplateStreamTransformer(tmpl, inStream, outStream) + if err != nil { + return fmt.Errorf("template stream transform error: %v", err) + } + if err := tfm.Transform(); err != nil { + return fmt.Errorf("failed to transform: %v", err) + } + outputStr := outStream.String() + stdoutStr = outputStr + } + } + fmt.Fprintf(os.Stdout, "%s", stdoutStr) + } + stdErr, stdErrExists := resp.GetStdErr() + if stdErrExists { + fmt.Fprintf(os.Stderr, "%s", stdErr.String()) + } + return nil + case client.HTTP: + prep := anysdk.NewHTTPPreparator( + prov, + svc, + opStore, + map[int]map[string]interface{}{ + 0: payload.parameters, + }, + nil, + execCtx, + getLogger(), + ) + armoury, err := prep.BuildHTTPRequestCtx() if err != nil { return err } - fmt.Fprintf(os.Stdout, "%s", string(bodyBytes)) + for _, v := range armoury.GetRequestParams() { + argList := v.GetArgList() + + cc := anysdk.NewAnySdkClientConfigurator( + payload.rtCtx, + prov.GetName(), + ) + response, apiErr := anysdk.CallFromSignature( + cc, payload.rtCtx, authCtx, authCtx.Type, false, os.Stderr, prov, anysdk.NewAnySdkOpStoreDesignation(opStore), argList) + if apiErr != nil { + return err + } + httpResponse, httpResponseErr := response.GetHttpResponse() + if httpResponseErr != nil { + return httpResponseErr + } + defer httpResponse.Body.Close() + bodyBytes, err := io.ReadAll(httpResponse.Body) + if err != nil { + return err + } + fmt.Fprintf(os.Stdout, "%s", string(bodyBytes)) + } + return nil + default: + return fmt.Errorf("protocol type = '%v' not supported", protocolType) } - return nil } func transformOpenapiStackqlAuthToLocal(authDTO anysdk.AuthDTO) *dto.AuthCtx { @@ -345,13 +305,13 @@ var queryCmd = &cobra.Command{ provStr := prov.GetName() - printErrorAndExitOneIfError(err) + protocolType, protocolTypeErr := prov.GetProtocolType() - gp := newGenericProvider(runtimeCtx, prov) + printErrorAndExitOneIfError(protocolTypeErr) auth, isAuthPresent := payload.auth[provStr] - if !isAuthPresent { + if !isAuthPresent && protocolType == client.HTTP { authDTO, isAuthPresent := prov.GetAuth() if !isAuthPresent { printErrorAndExitOneIfError(fmt.Errorf("auth not present")) @@ -360,7 +320,6 @@ var queryCmd = &cobra.Command{ } err = runQueryCommand( - gp, auth, payload, ) diff --git a/cmd/argparse/read.go b/cmd/argparse/read.go index 4fbf95b..0c9b60c 100644 --- a/cmd/argparse/read.go +++ b/cmd/argparse/read.go @@ -53,8 +53,7 @@ var execCmd = &cobra.Command{ func runReadCommand(rtCtx dto.RuntimeCtx, arg string) { b, err := os.ReadFile(arg) printErrorAndExitOneIfError(err) - l := anysdk.NewLoader() - svc, err := l.LoadFromBytes(b) + svc, err := anysdk.ReadService(b) printErrorAndExitOneIfError(err) printErrorAndExitOneIfNil(svc, "doc parse gave me doughnuts!!!\n\n") fmt.Fprintf(os.Stdout, "\nsuccessfully parsed svc = '%s'\n", svc.GetName()) diff --git a/docs/cli.md b/docs/cli.md index 5ed9c69..093ce60 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -28,6 +28,7 @@ The `const` command is very much a trivial "Hello World": ### Query +HTTP Provider: ```bash @@ -52,3 +53,36 @@ export GOOGLE_CREDENTIALS="$(cat cicd/keys/google-ro-credentials.json)" | jq -r '.items[].id' ``` + +Local templated provider mutation: + +```bash + +./build/anysdk query \ + --svc-file-path="test/registry/src/local_openssl/v0.1.0/services/keys.yaml" \ + --prov-file-path="test/registry/src/local_openssl/v0.1.0/provider.yaml" \ + --resource rsa \ + --method create_key_pair \ + --parameters '{ + "config_file": "test/openssl/openssl.cnf", + "key_out_file": "test/tmp/key.pem", + "cert_out_file": "test/tmp/cert.pem", + "days": 90 + }' + +``` + +Local templated provider selection: + +```bash + +./build/anysdk query \ + --svc-file-path="test/registry/src/local_openssl/v0.1.0/services/keys.yaml" \ + --prov-file-path="test/registry/src/local_openssl/v0.1.0/provider.yaml" \ + --resource x509 \ + --method describe_certificate \ + --parameters '{ + "cert_file": "test/tmp/cert.pem" + }' + +``` diff --git a/docs/protocol_agnostic/templating-snippet.go b/docs/protocol_agnostic/templating-snippet.go new file mode 100644 index 0000000..a84e7c9 --- /dev/null +++ b/docs/protocol_agnostic/templating-snippet.go @@ -0,0 +1,36 @@ +package main + +import ( + "encoding/json" + "log" + "os" + "text/template" +) + +func main() { + // Define a template. + const letter = ` +{{ or .parameters.executable "openssl" }} req -x509 -keyout {{ .parameters.key_out_file }} -out {{ .parameters.cert_out_file }} -config {{ .parameters.config_file }} -days {{ .parameters.days }} +` + + var recipientStr = `[ + {"parameters": { "key_out_file": "test/key.pem", "cert_out_file": "test/cert.pem", "config_file": "test/openssl.conf", "days": 365 }} + ]` + + var recipients []map[string]interface{} + if err := json.Unmarshal([]byte(recipientStr), &recipients); err != nil { + panic(err) + } + + // Create a new template and parse the letter into it. + t := template.Must(template.New("letter").Parse(letter)) + + // Execute the template for each recipient. + for _, r := range recipients { + err := t.Execute(os.Stdout, r) + if err != nil { + log.Println("executing template:", err) + } + } + +} diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..0898657 --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,88 @@ +package client + +import ( + "fmt" + "net/http" + + "github.com/stackql/any-sdk/pkg/dto" +) + +/* + +We model arbiitrary remote and local interfaces like C functions, composed of: + +- Name. +- Argument list. +- Return type. + +In the fist instance, support for stateful TCP/IP protocols, unix sockets and standard os primitives for spawned process communication are targeted. + +Aspirationally, socketless (network) protocols, sundry inter-process communication mechanisms are further targets. + +*/ + +type ClientProtocolType int + +const ( + ClientProtocolTypeHTTP string = "http" + ClientProtocolTypeLocalTemplated string = "local_templated" +) + +const ( + HTTP ClientProtocolType = iota + LocalTemplated + Disallowed +) + +func ClientProtocolTypeFromString(s string) (ClientProtocolType, error) { + switch s { + case ClientProtocolTypeHTTP: + return HTTP, nil + case ClientProtocolTypeLocalTemplated: + return LocalTemplated, nil + default: + return Disallowed, fmt.Errorf("unsupported protocol type: %s", s) + } +} + +type AnySdkResponse interface { + IsErroneous() bool + GetHttpResponse() (*http.Response, error) +} + +type AnySdkDesignation interface { + GetDesignation() (interface{}, bool) +} + +type AnySdkArg interface { + GetArg() (interface{}, bool) +} + +type AnySdkArgList interface { + GetArgs() []AnySdkArg + GetProtocolType() ClientProtocolType +} + +type AnySdkInvocation interface { + GetDesignation() (AnySdkDesignation, bool) + GetArgs() (AnySdkArgList, bool) +} + +type AnySdkClient interface { + Do(AnySdkDesignation, AnySdkArgList) (AnySdkResponse, error) +} + +type AnySdkClientConfigurator interface { + Auth( + authCtx *dto.AuthCtx, + authTypeRequested string, + enforceRevokeFirst bool, + ) (AnySdkClient, error) + InferAuthType(authCtx dto.AuthCtx, authTypeRequested string) string +} + +type ClientConfiguratorInput interface { + GetAuthContext() *dto.AuthCtx + GetAuthType() string + GetEnforceRevokeFirst() bool +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index f7ecc38..35a6261 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -57,6 +57,10 @@ const ( LimitsIndirectMaxChainLength int = 1 ) +const ( + NilTabulationName string = "__any__sdk__nil__" +) + const ( ReversalStreamAlias string = "reversal_stream" ReversalStreamID int64 = -1 diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index 6148961..1996df0 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -6,9 +6,11 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "text/template" + "github.com/stackql/any-sdk/pkg/client" "github.com/stackql/any-sdk/pkg/jsonpath" ) @@ -17,7 +19,7 @@ var ( ) func NewStandardGQLReader( - httpClient *http.Client, + anySdkClient client.AnySdkClient, request *http.Request, httpPageLimit int, baseQuery string, @@ -31,7 +33,7 @@ func NewStandardGQLReader( return nil, err } rv := &StandardGQLReader{ - httpClient: httpClient, + anySdkClient: anySdkClient, baseQuery: baseQuery, httpPageLimit: httpPageLimit, constInput: constInput, @@ -53,7 +55,7 @@ type StandardGQLReader struct { baseQuery string constInput map[string]interface{} iterativeInput map[string]interface{} - httpClient *http.Client + anySdkClient client.AnySdkClient httpPageLimit int queryTemplate *template.Template responseJsonPath string @@ -62,6 +64,52 @@ type StandardGQLReader struct { pageCount int } +type anySdkGraphQLHTTPDesignation struct { + url *url.URL +} + +func newAnySdkGraphQLHTTPDesignation(url *url.URL) client.AnySdkDesignation { + return &anySdkGraphQLHTTPDesignation{ + url: url, + } +} + +func (hd *anySdkGraphQLHTTPDesignation) GetDesignation() (interface{}, bool) { + return hd.url, hd.url != nil +} + +type graphqlAnySdkArgList struct { + args []client.AnySdkArg +} + +func (al *graphqlAnySdkArgList) GetArgs() []client.AnySdkArg { + return al.args +} + +func (al *graphqlAnySdkArgList) GetProtocolType() client.ClientProtocolType { + return client.HTTP +} + +func newGraphqlAnySdkArgList(args ...client.AnySdkArg) client.AnySdkArgList { + return &graphqlAnySdkArgList{ + args: args, + } +} + +type anySdkHTTPArg struct { + arg *http.Request +} + +func (ha *anySdkHTTPArg) GetArg() (interface{}, bool) { + return ha.arg, ha.arg != nil +} + +func newAnySdkHTTPArg(arg *http.Request) client.AnySdkArg { + return &anySdkHTTPArg{ + arg: arg, + } +} + func (gq *StandardGQLReader) Read() ([]map[string]interface{}, error) { if gq.httpPageLimit > 0 && gq.pageCount >= gq.httpPageLimit { return nil, io.EOF @@ -74,13 +122,20 @@ func (gq *StandardGQLReader) Read() ([]map[string]interface{}, error) { req.Body = rb req.URL.RawQuery = "" req.Header.Set("Content-Type", "application/json") - r, err := gq.httpClient.Do(req) + r, err := gq.anySdkClient.Do( + newAnySdkGraphQLHTTPDesignation(req.URL), + newGraphqlAnySdkArgList(newAnySdkHTTPArg(req)), + ) if err != nil { return nil, err } + httpResponse, httpResponseErr := r.GetHttpResponse() + if httpResponseErr != nil { + return nil, httpResponseErr + } gq.pageCount++ var target map[string]interface{} - err = json.NewDecoder(r.Body).Decode(&target) + err = json.NewDecoder(httpResponse.Body).Decode(&target) if err != nil { return nil, err } diff --git a/pkg/local_template_executor/local_template_executor.go b/pkg/local_template_executor/local_template_executor.go new file mode 100644 index 0000000..0c6b575 --- /dev/null +++ b/pkg/local_template_executor/local_template_executor.go @@ -0,0 +1,100 @@ +package local_template_executor + +import ( + "bytes" + "io" + "os/exec" + "text/template" +) + +var ( + _ io.Reader = &bytes.Buffer{} + _ io.Writer = &bytes.Buffer{} +) + +type ExecutionResponse interface { + GetStdOut() (*bytes.Buffer, bool) + GetStdErr() (*bytes.Buffer, bool) + GetExitCode() int +} + +type Executor interface { + Execute(map[string]any) (ExecutionResponse, error) +} + +func NewLocalTemplateExecutor(commandName string, commandArgs []string, stdInStream *bytes.Buffer) Executor { + return &localTemplateExecutor{ + stdInStream: stdInStream, + commandName: commandName, + commandArgs: commandArgs, + stdOutBuffer: &bytes.Buffer{}, + stdErrBuffer: &bytes.Buffer{}, + } +} + +type standardExecutionResponse struct { + stdOut *bytes.Buffer + stdErr *bytes.Buffer + exitCode int +} + +func (ser *standardExecutionResponse) GetStdOut() (*bytes.Buffer, bool) { + return ser.stdOut, ser.stdOut != nil +} + +func (ser *standardExecutionResponse) GetStdErr() (*bytes.Buffer, bool) { + return ser.stdErr, ser.stdErr != nil +} + +func (ser *standardExecutionResponse) GetExitCode() int { + return ser.exitCode +} + +type localTemplateExecutor struct { + // Path to the template file. + stdInStream *bytes.Buffer + commandName string + commandArgs []string + stdOutBuffer *bytes.Buffer + stdErrBuffer *bytes.Buffer +} + +func (lt *localTemplateExecutor) Execute(context map[string]any) (ExecutionResponse, error) { + // Execute the template file. + cmdTpl, cmdTplErr := template.New("letter").Parse(lt.commandName) + if cmdTplErr != nil { + return nil, cmdTplErr + } + var cmdBuffer bytes.Buffer + err := cmdTpl.Execute(&cmdBuffer, context) + if err != nil { + return nil, err + } + cmdString := cmdBuffer.String() + var commandStrArgs []string + for _, arg := range lt.commandArgs { + cmdTpl, cmdTplErr = template.New("letter").Parse(arg) + if cmdTplErr != nil { + return nil, cmdTplErr + } + cmdBuffer.Reset() + err = cmdTpl.Execute(&cmdBuffer, context) + if err != nil { + return nil, err + } + commandStrArgs = append(commandStrArgs, cmdBuffer.String()) + } + cmd := exec.Command(cmdString, commandStrArgs...) + // cmd.Stdin = lt.stdInStream + cmd.Stdout = lt.stdOutBuffer + cmd.Stderr = lt.stdErrBuffer + err = cmd.Run() + if err != nil { + return nil, err + } + return &standardExecutionResponse{ + stdOut: lt.stdOutBuffer, + stdErr: lt.stdErrBuffer, + exitCode: cmd.ProcessState.ExitCode(), + }, nil +} diff --git a/pkg/maths/common_maths.go b/pkg/maths/common_maths.go new file mode 100644 index 0000000..21a8a86 --- /dev/null +++ b/pkg/maths/common_maths.go @@ -0,0 +1,55 @@ +package maths + +// Pretty much transcribed from https://stackoverflow.com/a/147539 +// LcmMultiple returns the greatest common multiple of a and b... + +func Gcd(a, b int) int { + // Return greatest common divisor using Euclid's Algorithm. + for b != 0 { + a, b = b, a%b + } + return a +} + +func Lcm(a, b int) int { + // Return lowest common multiple. + return a * b / Gcd(a, b) +} + +func LcmMultiple(args ...int) int { + // """Return lcm of args.""" + if len(args) == 0 { + return 1 + } + rv := args[0] + for i := 1; i < len(args); i++ { + rv = Lcm(rv, args[i]) + } + return rv +} + +func CartesianProduct(args ...[]map[string]interface{}) []map[string]interface{} { + // """Return the Cartesian product of args.""" + if len(args) == 0 { + return []map[string]interface{}{} + } + rv := []map[string]interface{}{} + rv = append(rv, args[0]...) + for i := 1; i < len(args); i++ { + newRV := []map[string]interface{}{} + for _, row := range args[i] { + for _, existingRow := range rv { + newRow := map[string]interface{}{} + for k, v := range existingRow { + newRow[k] = v + } + for k, v := range row { + newRow[k] = v + } + newRV = append(newRV, newRow) + } + } + rv = newRV + } + return rv +} diff --git a/pkg/stream_transform/regexp_shorthand.go b/pkg/stream_transform/regexp_shorthand.go new file mode 100644 index 0000000..a4c4158 --- /dev/null +++ b/pkg/stream_transform/regexp_shorthand.go @@ -0,0 +1,42 @@ +package stream_transform + +import ( + "fmt" + "regexp" +) + +type RegexpShorthand interface { + GetFirstMatch(string, string) (string, error) + GetAllMatches(string, string) ([]string, error) +} + +type regexpShorthand struct { +} + +func (rs *regexpShorthand) GetFirstMatch(input string, pattern string) (string, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return "", err + } + match := re.FindStringSubmatch(input) + if len(match) < 2 { + return "", fmt.Errorf("no match found for pattern %q in input %q", pattern, input) + } + return match[1], nil +} + +func (rs *regexpShorthand) GetAllMatches(input string, pattern string) ([]string, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + match := re.FindStringSubmatch(input) + if len(match) < 2 { + return nil, fmt.Errorf("no match found for pattern %q in input %q", pattern, input) + } + return match[1:], nil +} + +func NewRegexpShorthand() RegexpShorthand { + return ®expShorthand{} +} diff --git a/pkg/stream_transform/stream_transform_test.go b/pkg/stream_transform/stream_transform_test.go new file mode 100644 index 0000000..38a8f42 --- /dev/null +++ b/pkg/stream_transform/stream_transform_test.go @@ -0,0 +1,291 @@ +package stream_transform_test + +import ( + "bytes" + "fmt" + "io" + "testing" + + . "github.com/stackql/any-sdk/pkg/stream_transform" +) + +var ( + _ io.Reader = &bytes.Buffer{} + _ io.Writer = &bytes.Buffer{} + jsonExample = `{ + "animals": [ + {"name": "Platypus", "order": "Monotremata", "votes": 1, "bank_balance": 100.0}, + {"name": "Quokka", "order": "Diprotodontia", "votes": 3, "bank_balance": 200.0}, + {"name": "Quoll", "order": "Dasyuromorphia", "votes": 2, "bank_balance": 300.0, "premierships": [1993, 2000]} + ], + "meta": { + "institution": "University of Tasmania", + "total_votes": 6, + "total_bank_balance": 600.0 + } + }` + xmlExample = ` + + + + Platypus + Monotremata + 1 + 100.0 + + + Quokka + Diprotodontia + 3 + 200.0 + + + Quoll + Dasyuromorphia + 2 + 300.0 + + 1993 + 2000 + + + + + University of Tasmania + 6 + 600.0 + + + ` + yamlExample = `--- +animals: + - name: Platypus + order: Monotremata + votes: 1 + bank_balance: 100.0 + - name: Quokka + order: Diprotodontia + votes: 3 + bank_balance: 200.0 + - name: Quoll + order: Dasyuromorphia + votes: 2 + bank_balance: 300.0 + premierships: + - 1993 + - 2000 +meta: + institution: University of Tasmania + total_votes: 6 + total_bank_balance: 600.0 +` + jsonTmpl = ` + {{- $s := separator ", " -}} + [ + {{- range $idx, $animal := $.animals -}} + {{- call $s}}{"name": "{{ $animal.name }}", "democratic_votes": {{ $animal.votes }}} + {{- end -}} + ]` + xmlTmpl = `[{ "name": "{{- getXPath . "/root/animals/animal/name" }}"}]` + expectedJsonOutput = `[{"name": "Platypus", "democratic_votes": 1}, {"name": "Quokka", "democratic_votes": 3}, {"name": "Quoll", "democratic_votes": 2}]` + openSSLCertTextOutput = `Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 53:f4:3b:da:df:42:7b:bf:c3:14:08:9a:69:6d:a1:7b:47:2d:8b:8a + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=AU, ST=VIC, L=Melbourne, O=StackQL, OU=Core Functions, CN=127.0.0.1, emailAddress=krimmer@stackql.io + Validity + Not Before: Mar 22 02:50:46 2025 GMT + Not After : Jun 20 02:50:46 2025 GMT + Subject: C=AU, ST=VIC, L=Melbourne, O=StackQL, OU=Core Functions, CN=127.0.0.1, emailAddress=krimmer@stackql.io + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:b8:55:1a:d3:ac:e8:5c:19:a9:24:be:ee:36:47: + 79:92:c9:56:e1:40:92:40:51:13:03:7b:7b:b4:00: + ac:70:e3:80:bf:8a:0a:86:d8:27:0d:97:7f:d9:4c: + 4d:fd:b0:4c:9c:d7:22:a7:5b:1e:c2:4c:be:bf:e5: + 74:b7:e7:00:a9:ad:0e:a5:02:fd:c8:62:df:8b:c2: + 87:de:eb:54:7f:c4:69:ae:e1:f2:e5:e6:2b:9f:34: + 0c:24:7e:3c:b4:b1:75:11:1c:16:c7:5e:0d:32:b8: + 7e:bd:3c:0c:7d:0d:5f:84:55:9d:be:16:e3:bd:04: + c4:ba:9c:cd:8e:a9:56:a9:67:7c:79:60:22:c7:4c: + 46:de:52:97:2d:fc:7e:67:3b:c5:ae:3e:9c:c3:c0: + be:b6:a1:82:be:5b:f5:1b:f2:a9:87:ea:ad:0d:bf: + b9:21:dc:dd:cf:70:d3:89:d0:8f:ab:f5:9c:67:3f: + d8:e3:93:80:55:3c:46:08:1a:90:20:40:2f:84:e9: + 7d:b7:b4:4f:0b:80:80:b5:cc:51:92:6d:d0:12:f5: + e1:aa:2a:a7:3f:1c:2b:23:6b:92:b1:ad:cd:35:e2: + 98:6e:f1:e9:60:85:60:aa:53:41:f7:91:b2:ac:b9: + 83:8f:ca:44:0a:d0:53:4f:dc:15:89:54:1d:43:85: + 67:cf:f8:da:39:09:02:8c:0d:3a:e8:f0:4e:a2:1d: + 5a:54:d6:5f:87:9a:3a:11:4e:ad:85:4a:b6:f7:c2: + e3:e9:e0:d0:10:fa:3d:c2:59:98:80:0c:b7:40:71: + 05:df:f5:72:a6:54:a2:5b:82:39:58:dd:17:72:44: + b1:15:03:f2:7a:26:0a:e0:db:83:1a:51:d1:1c:37: + c5:8d:dc:1e:72:b5:1a:d7:24:fc:4c:c6:17:84:54: + a4:65:3a:44:ec:11:dd:fc:ca:fd:20:fc:f7:25:01: + 5c:38:af:66:bf:d8:c2:47:53:a6:e9:cb:52:32:8f: + d5:10:45:7c:0c:c1:54:3c:3a:e6:eb:50:22:b6:f2: + 66:94:a0:1b:4e:c1:3d:32:3d:d4:a3:09:97:ed:aa: + d9:13:e4:5f:64:b9:d0:5f:ca:6b:b7:6c:98:8c:80: + 86:26:6f:24:d5:19:de:11:29:e1:91:a0:45:03:7b: + fc:38:e2:a8:b3:c5:34:e2:e3:00:79:33:6d:57:1a: + 1e:e7:a6:a9:3d:07:c5:6c:b7:67:c6:f5:db:d0:4d: + 5d:8c:7c:06:b7:33:80:14:5a:b4:cf:43:4c:05:cf: + 61:80:85:7a:46:4c:e7:7c:0a:00:dd:ce:d6:cf:13: + 1e:28:a1:6b:66:17:fb:7f:77:83:10:20:49:f9:3b: + 71:70:31 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + DB:F5:C4:59:02:8E:D5:3E:58:16:E3:C6:AD:78:A9:00:68:16:8E:DD + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + a2:4e:1b:97:c1:ce:7a:16:8b:1b:bb:4a:17:f9:9f:bd:95:31: + af:84:05:51:cf:52:d1:17:96:02:87:f0:26:0b:0d:40:85:fb: + 44:d6:0c:76:3d:60:fb:f7:c0:f6:3f:7a:64:1d:6e:82:01:2d: + 6e:aa:46:dd:3b:af:34:e4:cb:ca:50:78:08:2f:98:e8:ed:c0: + 1e:65:71:14:c2:1f:e0:cb:d7:e9:43:5b:b6:60:c5:de:d3:65: + 2e:b1:51:31:25:28:73:fd:a8:96:e1:b0:a9:ef:b3:4d:dd:2c: + 89:9a:80:38:59:54:55:52:a6:8e:9f:1b:50:c1:e1:8b:44:66: + dc:43:b8:eb:ac:d6:aa:e9:17:7b:b0:61:1b:41:65:83:23:9c: + 0a:b2:9f:1b:c4:e9:06:a3:ad:43:f2:e3:4a:3c:29:6d:c6:72: + 03:59:79:87:f4:de:86:46:2c:cc:80:c1:bd:bd:7d:f6:41:fa: + 5f:e3:6b:c4:34:ed:10:ae:59:1f:bb:c4:c3:70:22:c9:ee:ef: + 16:e2:fb:17:40:ca:71:9c:91:76:8e:00:bd:4b:d3:6c:63:f1: + 30:9f:3c:6c:9c:ad:e7:c4:37:34:9e:ec:a2:50:d3:c5:44:5b: + af:27:c3:cd:70:ee:b3:ea:1c:aa:6f:cd:69:85:f5:9c:4b:e9: + b5:68:12:c8:59:78:84:1f:5f:51:59:e3:38:1a:17:8b:76:54: + f2:dd:dc:e9:d5:5a:a0:45:5a:21:e3:0b:03:05:90:ab:a7:57: + d9:e5:62:2a:0d:ed:85:60:00:d6:b2:e0:75:4b:90:4d:a7:03: + 66:1c:29:12:14:8c:3f:06:15:d6:01:62:e2:49:d3:92:e6:cf: + 13:dd:5e:3a:8b:2f:10:8d:ca:27:d9:33:bc:0d:11:17:5f:05: + 8e:ae:f5:a0:12:36:07:8f:f3:e3:22:f9:d0:43:68:40:17:2c: + 9b:7e:bc:b9:5e:8e:b2:49:45:78:e8:ba:b9:85:4d:dd:e7:5e: + 27:e9:da:14:be:29:a4:2b:02:53:83:a1:11:08:43:f5:4e:9d: + 16:84:f2:64:a8:e3:49:d7:6a:dd:33:32:49:a8:b6:cf:8e:14: + d1:2b:e2:61:8c:ec:f1:3a:d7:8d:ee:74:0e:26:71:e9:4e:4f: + 4c:66:4a:5a:ee:8f:56:69:1a:22:11:cd:e1:b2:fa:69:3d:d9: + 07:51:74:af:bb:12:22:d8:4b:43:aa:41:3b:de:6d:13:45:5c: + c3:c6:9a:54:5a:c4:90:40:e1:c7:df:9d:3b:da:15:2e:0d:d2: + 73:21:b4:62:a7:4a:3f:8a:66:8e:02:38:d5:50:5c:d4:96:86: + a5:66:21:c3:39:6f:54:cc +` +) + +func TestSimpleStreamTransform(t *testing.T) { + input := fmt.Sprintf(`"Hello, %s!"`, "World") + t.Log("TestSimpleStream") + tmpl := `{{.}}` + inStream := NewJSONReader(bytes.NewBufferString(input)) + outStream := bytes.NewBuffer(nil) + tfm, err := NewTemplateStreamTransformer(tmpl, inStream, outStream) + if err != nil { + t.Fatalf("failed to create transformer: %v", err) + } + if err := tfm.Transform(); err != nil { + t.Fatalf("failed to transform: %v", err) + } + outputStr := outStream.String() + if outputStr != "Hello, World!" { + t.Fatalf("unexpected output: %s", outStream.String()) + } +} + +func TestMeaningfulStreamTransform(t *testing.T) { + input := jsonExample + t.Log("TestSimpleStream") + tmpl := jsonTmpl + inStream := NewJSONReader(bytes.NewBufferString(input)) + outStream := bytes.NewBuffer(nil) + tfm, err := NewTemplateStreamTransformer(tmpl, inStream, outStream) + if err != nil { + t.Fatalf("failed to create transformer: %v", err) + } + if err := tfm.Transform(); err != nil { + t.Fatalf("failed to transform: %v", err) + } + outputStr := outStream.String() + if outputStr != expectedJsonOutput { + t.Fatalf("unexpected output: '%s' != '%s'", outputStr, expectedJsonOutput) + } +} + +func TestSimpleXMLStreamTransform(t *testing.T) { + input := xmlExample + t.Log("v") + tmpl := xmlTmpl + inStream := NewTextReader(bytes.NewBufferString(input)) + outStream := bytes.NewBuffer(nil) + tfm, err := NewTemplateStreamTransformer(tmpl, inStream, outStream) + if err != nil { + t.Fatalf("failed to create transformer: %v", err) + } + if err := tfm.Transform(); err != nil { + t.Fatalf("failed to transform: %v", err) + } + outputStr := outStream.String() + expected := `[{ "name": "Platypus"}]` + if outputStr != expected { + t.Fatalf("unexpected output: '%s' != '%s'", outputStr, expected) + } +} + +func TestMeaningfulXMLStreamTransform(t *testing.T) { + input := xmlExample + t.Log("TestMeaningfulXMLStreamTransform") + tmpl := ` + {{- $s := separator ", " -}} + [ + {{- $animals := getXPathAllOuter . "/root/animals/animal" -}} + {{- range $animal := $animals -}} + {{- call $s -}} + {{- $animalName := getXPath $animal "/animal/name" -}} + {{- $animalVotes := getXPath $animal "/animal/votes" -}} + {"name": "{{ $animalName }}", "democratic_votes": {{ $animalVotes }}} + {{- end -}} + ]` + inStream := NewTextReader(bytes.NewBufferString(input)) + outStream := bytes.NewBuffer(nil) + tfm, err := NewTemplateStreamTransformer(tmpl, inStream, outStream) + if err != nil { + t.Fatalf("failed to create transformer: %v", err) + } + if err := tfm.Transform(); err != nil { + t.Fatalf("failed to transform: %v", err) + } + outputStr := outStream.String() + expected := `[{"name": "Platypus", "democratic_votes": 1}, {"name": "Quokka", "democratic_votes": 3}, {"name": "Quoll", "democratic_votes": 2}]` + if outputStr != expected { + t.Fatalf("unexpected output: '%s' != '%s'", outputStr, expected) + } +} + +func TestOpensslCertTextStreamTransform(t *testing.T) { + input := openSSLCertTextOutput + t.Log("TestOpensslCertTextStreamTransform") + tmpl := ` + {{- $s := separator ", " -}} + {{- $root := . -}} + {{- $pubKeyAlgo := getRegexpFirstMatch $root "Public Key Algorithm: (?.*)" -}} + {{- $notBefore := getRegexpFirstMatch $root "Not Before: (.*)" -}} + {{- $notAfter := getRegexpFirstMatch $root "Not After(?:[ ]*): (.*)" -}} + { "type": "x509", "public_key_algorithm": "{{ $pubKeyAlgo }}", "not_before": "{{ $notBefore }}", "not_after": "{{ $notAfter }}"}` + inStream := NewTextReader(bytes.NewBufferString(input)) + outStream := bytes.NewBuffer(nil) + tfm, err := NewTemplateStreamTransformer(tmpl, inStream, outStream) + if err != nil { + t.Fatalf("failed to create transformer: %v", err) + } + if err := tfm.Transform(); err != nil { + t.Fatalf("failed to transform: %v", err) + } + outputStr := outStream.String() + expected := `{ "type": "x509", "public_key_algorithm": "rsaEncryption", "not_before": "Mar 22 02:50:46 2025 GMT", "not_after": "Jun 20 02:50:46 2025 GMT"}` + if outputStr != expected { + t.Fatalf("unexpected output: '%s' != '%s'", outputStr, expected) + } +} diff --git a/pkg/stream_transform/template_stream_transform.go b/pkg/stream_transform/template_stream_transform.go new file mode 100644 index 0000000..47a2f49 --- /dev/null +++ b/pkg/stream_transform/template_stream_transform.go @@ -0,0 +1,156 @@ +package stream_transform + +import ( + "bytes" + "encoding/json" + "io" + "text/template" +) + +// full acknowledgment to https://stackoverflow.com/a/42663928 +func separator(s string) func() string { + i := -1 + return func() string { + i++ + if i == 0 { + return "" + } + return s + } +} + +func getXPathInner(xml string, path string) (string, error) { + ss := NewXMLStringShorthand() + return ss.GetFirstInner(xml, path) +} + +func getRegexpFirstMatch(input string, pattern string) (string, error) { + rs := NewRegexpShorthand() + return rs.GetFirstMatch(input, pattern) +} + +func getRegexpAllMatches(input string, pattern string) ([]string, error) { + rs := NewRegexpShorthand() + return rs.GetAllMatches(input, pattern) +} + +func getXPathAllOuter(xml string, path string) ([]string, error) { + ss := NewXMLStringShorthand() + return ss.GetAllFull(xml, path) +} + +type ObjectReader interface { + Read() (interface{}, error) +} + +type jsonReader struct { + inStream io.Reader +} + +func NewJSONReader(inStream io.Reader) ObjectReader { + return &jsonReader{ + inStream: inStream, + } +} + +func (jr *jsonReader) Read() (interface{}, error) { + var v interface{} + err := json.NewDecoder(jr.inStream).Decode(&v) + return v, err +} + +type textReader struct { + inStream io.Reader +} + +func NewTextReader(inStream io.Reader) ObjectReader { + return &textReader{ + inStream: inStream, + } +} + +func (tr *textReader) Read() (interface{}, error) { + var buf bytes.Buffer + _, err := buf.ReadFrom(tr.inStream) + if err != nil { + return "", err + } + rv := buf.String() + return rv, io.EOF +} + +func jsonMapFromString(s string) (map[string]interface{}, error) { + var v map[string]interface{} + err := json.Unmarshal([]byte(s), &v) + return v, err +} + +type StreamTransformer interface { + Transform() error +} + +type templateStreamTransfomer struct { + tpl *template.Template + inStream ObjectReader + outStream io.Writer +} + +func NewTemplateStreamTransformer( + tplStr string, + inStream ObjectReader, + outStream io.Writer, +) (StreamTransformer, error) { + return newTemplateStreamTransformer(tplStr, inStream, outStream) +} + +func newTemplateStreamTransformer( + tplStr string, + inStream ObjectReader, + outStream io.Writer, +) (StreamTransformer, error) { + tpl, tplErr := template.New("__stream_tfm__").Funcs(template.FuncMap{ + "separator": separator, + "jsonMapFromString": jsonMapFromString, + "getXPath": getXPathInner, + "getXPathAllOuter": getXPathAllOuter, + "getRegexpFirstMatch": getRegexpFirstMatch, + "getRegexpAllMatches": getRegexpAllMatches, + }).Parse(tplStr) + if tplErr != nil { + return nil, tplErr + } + if outStream == nil { + outStream = bytes.NewBuffer(nil) + } + return &templateStreamTransfomer{ + tpl: tpl, + inStream: inStream, + outStream: outStream, + }, nil +} + +func (tst *templateStreamTransfomer) Transform() error { + for { + obj, readErr := tst.inStream.Read() + if obj == nil { + if readErr != nil && readErr != io.EOF { + return readErr + } + break + } + execErr := tst.tpl.Execute(tst.outStream, obj) + if readErr == io.EOF { + break + } + if readErr != nil { + return readErr + } + if execErr == io.EOF { + break + } + if execErr != nil { + return execErr + } + } + return nil +} diff --git a/pkg/stream_transform/xml_shorthand.go b/pkg/stream_transform/xml_shorthand.go new file mode 100644 index 0000000..d76578f --- /dev/null +++ b/pkg/stream_transform/xml_shorthand.go @@ -0,0 +1,114 @@ +package stream_transform + +import ( + "io" + "strings" + + "github.com/antchfx/xmlquery" +) + +var ( + _ *xmlquery.Node = (*xmlquery.Node)(nil) +) + +type XMLStringShorthand interface { + GetFirstFull(string, string) (string, error) + GetAllFull(string, string) ([]string, error) + GetFirstInner(string, string) (string, error) + GetAllInner(string, string) ([]string, error) +} + +func NewXMLStringShorthand() XMLStringShorthand { + return &xmlStringShorthand{ + shorthand: NewXMLShorthand(), + } +} + +type xmlStringShorthand struct { + shorthand XMLShorthand +} + +func (xs *xmlStringShorthand) GetFirstFull(input string, path string) (string, error) { + return xs.shorthand.GetFirstFull(strings.NewReader(input), path) +} + +func (xs *xmlStringShorthand) GetAllFull(input string, path string) ([]string, error) { + return xs.shorthand.GetAllFull(strings.NewReader(input), path) +} + +func (xs *xmlStringShorthand) GetFirstInner(input string, path string) (string, error) { + return xs.shorthand.GetFirstInner(strings.NewReader(input), path) +} + +func (xs *xmlStringShorthand) GetAllInner(input string, path string) ([]string, error) { + return xs.shorthand.GetAllInner(strings.NewReader(input), path) +} + +type XMLShorthand interface { + GetFirstFull(io.Reader, string) (string, error) + GetAllFull(io.Reader, string) ([]string, error) + GetFirstInner(io.Reader, string) (string, error) + GetAllInner(io.Reader, string) ([]string, error) +} + +func NewXMLShorthand() XMLShorthand { + return &xmlShorthand{} +} + +type xmlShorthand struct{} + +func (xs *xmlShorthand) GetFirstFull(input io.Reader, path string) (string, error) { + node, err := xs.getFirst(input, path) + if err != nil { + return "", err + } + return node.OutputXML(true), nil +} + +func (xs *xmlShorthand) GetAllFull(input io.Reader, path string) ([]string, error) { + nodes, err := xs.getAll(input, path) + if err != nil { + return nil, err + } + var rv []string + for _, node := range nodes { + rv = append(rv, node.OutputXML(true)) + } + return rv, nil +} + +func (xs *xmlShorthand) GetFirstInner(input io.Reader, path string) (string, error) { + node, err := xs.getFirst(input, path) + if err != nil { + return "", err + } + return node.InnerText(), nil +} + +func (xs *xmlShorthand) GetAllInner(input io.Reader, path string) ([]string, error) { + nodes, err := xs.getAll(input, path) + if err != nil { + return nil, err + } + var rv []string + for _, node := range nodes { + rv = append(rv, node.InnerText()) + } + return rv, nil +} + +func (xs *xmlShorthand) getFirst(input io.Reader, path string) (*xmlquery.Node, error) { + doc, err := xmlquery.Parse(input) + if err != nil { + return nil, err + } + return xmlquery.Query(doc, path) +} + +func (xs *xmlShorthand) getAll(input io.Reader, path string) ([]*xmlquery.Node, error) { + doc, err := xmlquery.Parse(input) + if err != nil { + return nil, err + } + return xmlquery.QueryAll(doc, path) +} diff --git a/pkg/streaming/map_stream_collection.go b/pkg/streaming/map_stream_collection.go new file mode 100644 index 0000000..38c16fa --- /dev/null +++ b/pkg/streaming/map_stream_collection.go @@ -0,0 +1,89 @@ +package streaming + +import ( + "errors" + "fmt" + "io" + "strings" + + "github.com/stackql/any-sdk/pkg/maths" +) + +type MapStreamCollection interface { + MapStream + Push(MapStream) + Len() int +} + +func NewStandardMapStreamCollection() MapStreamCollection { + return &standardMapStreamCollection{} +} + +type standardMapStreamCollection struct { + store []map[string]interface{} + streams []MapStream +} + +func (sc *standardMapStreamCollection) Push(stream MapStream) { + sc.streams = append(sc.streams, stream) +} + +func (sc *standardMapStreamCollection) Write(input []map[string]interface{}) error { + // sc.store = append(sc.store, input...) + var errSlice []error + for _, stream := range sc.streams { + if err := stream.Write(input); err != nil { + errSlice = append(errSlice, err) + } + } + if len(errSlice) > 0 { + var sb strings.Builder + for _, err := range errSlice { + sb.WriteString(err.Error()) + sb.WriteString(";") + } + return fmt.Errorf(sb.String()) + } + return nil +} + +func (sc *standardMapStreamCollection) Len() int { + streamLen := len(sc.streams) + storeLen := len(sc.store) + if streamLen > storeLen { + return streamLen + } + return storeLen +} + +func (sc *standardMapStreamCollection) Read() ([]map[string]interface{}, error) { + var allOutputs [][]map[string]interface{} + maxLength := 0 + // var allLengths []int + storeLen := len(sc.store) + if storeLen > 0 { + // allLengths = append(allLengths, len(sc.store)) + maxLength = len(sc.store) + } + for _, stream := range sc.streams { + output, err := stream.Read() + if !errors.Is(err, io.EOF) { + return output, err + } + thisLen := len(output) + if thisLen == 0 { + continue + } + allOutputs = append(allOutputs, output) + // allLengths = append(allLengths, thisLen) + if thisLen > maxLength { + maxLength = thisLen + } + } + if maxLength == 0 { + return nil, io.EOF + } + // lcm := maths.LcmMultiple(allLengths...) + rv := maths.CartesianProduct(allOutputs...) + return rv, io.EOF +} diff --git a/pkg/xmlmap/xmlmap_test.go b/pkg/xmlmap/xmlmap_test.go index 0dc23f2..dc2a082 100644 --- a/pkg/xmlmap/xmlmap_test.go +++ b/pkg/xmlmap/xmlmap_test.go @@ -7,11 +7,10 @@ import ( "gotest.tools/assert" + "github.com/getkin/kin-openapi/openapi3" "github.com/stackql/any-sdk/pkg/fileutil" . "github.com/stackql/any-sdk/pkg/xmlmap" "github.com/stackql/any-sdk/test/pkg/testutil" - - "github.com/getkin/kin-openapi/openapi3" ) func getFileRoot(t *testing.T) string { diff --git a/test/openssl/openssl.cnf b/test/openssl/openssl.cnf new file mode 100644 index 0000000..3cac475 --- /dev/null +++ b/test/openssl/openssl.cnf @@ -0,0 +1,24 @@ +[ req ] +default_bits = 4096 +default_md = sha256 +prompt = no +encrypt_key = no +distinguished_name = pg_server +# stackql_ca +[ pg_ca ] +countryName = "AU" # C= +stateOrProvinceName = "VIC" # ST= +localityName = "Melbourne" # L= +organizationName = "StackQL" # O= +organizationalUnitName = "Core Functions" # OU= +commonName = "pg_ca" # CN= +emailAddress = "krimmer@stackql.io" # CN/emailAddress= +# stackql_server +[ pg_server ] +countryName = "AU" # C= +stateOrProvinceName = "VIC" # ST= +localityName = "Melbourne" # L= +organizationName = "StackQL" # O= +organizationalUnitName = "Core Functions" # OU= +commonName = "127.0.0.1" # CN= +emailAddress = "krimmer@stackql.io" # CN/emailAddress= diff --git a/test/registry/src/local_openssl/v0.1.0/provider.yaml b/test/registry/src/local_openssl/v0.1.0/provider.yaml new file mode 100644 index 0000000..9ad71db --- /dev/null +++ b/test/registry/src/local_openssl/v0.1.0/provider.yaml @@ -0,0 +1,18 @@ +id: local_openssl +name: local_openssl +version: v0.1.0 +protocolType: local_templated +providerServices: + keys: + description: Key management through openssl. + id: keys:v0.1.0 + name: keys + preferred: true + service: + $ref: local_openssl/v0.1.0/services/keys.yaml + title: openssl Key Management + version: v0.1.0 +openapi: 3.0.3 +config: + auth: + type: "null_auth" \ No newline at end of file diff --git a/test/registry/src/local_openssl/v0.1.0/services/keys.yaml b/test/registry/src/local_openssl/v0.1.0/services/keys.yaml new file mode 100644 index 0000000..fad7a8a --- /dev/null +++ b/test/registry/src/local_openssl/v0.1.0/services/keys.yaml @@ -0,0 +1,281 @@ +info: + version: 0.1.0 + title: Contrived Service for a Contrived Provider + contact: + name: Support + url: https://support.contrivedservice.contrivedprovider.com/contact +paths: {} +components: + schemas: + basic-error: + title: Basic Error + description: Basic Error + type: object + properties: + message: + type: string + documentation_url: + type: string + url: + type: string + status: + type: string + cert_display: + title: Key Display + type: object + properties: + type: + type: string + description: The key type + example: x509 + not_before: + type: string + description: Textual date representation of the key's not before date. + example: Mar 22 02:50:46 2025 GMT + not_after: + type: string + description: Textual date representation of the key's not after date. + example: Mar 22 02:50:46 2025 GMT + public_key_algorithm: + type: string + description: The public key algorithm used by the key. + example: rsaEncryption + page: + type: object + properties: + url: + type: string + description: The API address for accessing this Page resource. + format: uri + example: https://api.github.com/repos/github/hello-world/pages + status: + type: string + enum: + - built + - building + - errored + nullable: true + cname: + description: The Pages site's custom domain + example: example.com + type: string + nullable: true + protected_domain_state: + type: string + description: The state if the domain is verified + example: pending + nullable: true + enum: + - pending + - verified + - unverified + pending_domain_unverified_at: + type: string + description: The timestamp when a pending domain becomes unverified. + nullable: true + format: date-time + custom_404: + type: boolean + description: Whether the Page has a custom 404 page. + example: false + default: false + html_url: + type: string + description: The web address the Page can be accessed from. + format: uri + example: https://example.com + source: + $ref: '#/components/schemas/pages-source-hash' + public: + type: boolean + description: Whether the GitHub Pages site is publicly visible. If set to `true`, the site is accessible to anyone on the internet. If set to `false`, the site will only be accessible to users who have at least `read` access to the repository that published the site. + example: true + https_certificate: + $ref: '#/components/schemas/pages-https-certificate' + https_enforced: + type: boolean + description: Whether https is enabled on the domain + example: true + required: + - url + - status + - cname + - custom_404 + - public + pages-source-hash: + title: Pages Source Hash + type: object + properties: + branch: + type: string + path: + type: string + required: + - branch + - path + pages-https-certificate: + title: Pages Https Certificate + type: object + properties: + state: + type: string + enum: + - new + - authorization_created + - authorization_pending + - authorized + - authorization_revoked + - issued + - uploaded + - approved + - errored + - bad_authz + - destroy_pending + - dns_changed + example: approved + description: + type: string + example: Certificate is approved + domains: + type: array + items: + type: string + description: Array of the domain set and its alternate name (if it is configured) + example: + - example.com + - www.example.com + expires_at: + type: string + format: date + required: + - state + - description + - domains + parameters: + owner: + name: owner + in: path + required: true + schema: + type: string + repo: + name: repo + in: path + required: true + schema: + type: string + responses: + not_found: + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/basic-error' + securitySchemes: {} + callbacks: {} + requestBodies: {} +resources: + x509: + id: openssl_local.keys.x509 + name: x509 + title: x509 + methods: + describe_certificate: + summary: Describe an x509 certificate. + description: | + Describe an x509 certificate. + Classical usage: + openssl x509 -in test/tmp/cert.pem -noout -text + inline: + - '{{ or .parameters.executable "openssl" }}' + - x509 + - -in + - '{{ .parameters.cert_file }}' + - -noout + - -text + parameters: + cert_file: + in: inline + required: true + response: + schema_override: + title: Key Display + type: object + properties: + type: + type: string + description: The key type + example: x509 + not_before: + type: string + description: Textual date representation of the key's not before date. + example: Mar 22 02:50:46 2025 GMT + not_after: + type: string + description: Textual date representation of the key's not after date. + example: Mar 22 02:50:46 2025 GMT + public_key_algorithm: + type: string + description: The public key algorithm used by the key. + example: rsaEncryption + transform: + body: > + {{- $s := separator ", " -}} + {{- $root := . -}} + {{- $pubKeyAlgo := getRegexpFirstMatch $root "Public Key Algorithm: (?.*)" -}} + {{- $notBefore := getRegexpFirstMatch $root "Not Before: (.*)" -}} + {{- $notAfter := getRegexpFirstMatch $root "Not After(?:[ ]*): (.*)" -}} + { "type": "x509", "public_key_algorithm": "{{ $pubKeyAlgo }}", "not_before": "{{ $notBefore }}", "not_after": "{{ $notAfter }}"} + type: 'golang_template_v0.1.0' + + rsa: + id: openssl_local.keys.rsa + name: rsa + title: rsa + methods: + create_key_pair: + summary: Create a new RSA key pair. + description: | + Create a new RSA key pair. + Classical usage: + openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365 + inline: + - '{{ or .parameters.executable "openssl" }}' + - req + - -x509 + - -keyout + - '{{ .parameters.key_out_file }}' + - -out + - '{{ .parameters.cert_out_file }}' + - -config + - '{{ .parameters.config_file }}' + - -days + - '{{ or .parameters.days 365 }}' + parameters: + key_out_file: + in: inline + required: true + cert_out_file: + in: inline + required: true + config_file: + in: inline + required: true + days: + in: inline + required: false + executable: + in: inline + required: false + response: + mediaType: application/json + openAPIDocKey: '200' + sqlVerbs: + select: [] + insert: + - $ref: '#/components/x-stackQL-resources/rsa/methods/create_key_pair' + update: [] + delete: [] +openapi: 3.0.3 +servers: + - url: https://contrivedservice.contrivedprovider.com